diff --git a/0017-gcc-12-omnibus.patch b/0017-gcc-12-omnibus.patch deleted file mode 100644 index adc3300..0000000 --- a/0017-gcc-12-omnibus.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h.orig 2022-02-11 17:21:40.268627997 -0500 -+++ ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h 2022-02-11 17:21:57.155325437 -0500 -@@ -466,7 +466,7 @@ - - std::string print(int ident) override - { -- return std::string(0); -+ return std::string(""); - } - - void push_argument(base_statement* arg) diff --git a/0041-src-boost-boost-python-call_method.hpp.patch b/0041-src-boost-boost-python-call_method.hpp.patch deleted file mode 100644 index 7f908b2..0000000 --- a/0041-src-boost-boost-python-call_method.hpp.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ceph-18.2.3/src/boost/boost/python/call_method.hpp.orig 2024-06-07 11:14:58.579734749 -0400 -+++ ceph-18.2.3/src/boost/boost/python/call_method.hpp 2024-06-07 11:19:06.754520504 -0400 -@@ -59,7 +59,7 @@ - ) - { - PyObject* const result = -- PyEval_CallMethod( -+ PyObject_CallMethod( - self - , const_cast(name) - , const_cast("(" BOOST_PP_REPEAT_1ST(N, BOOST_PYTHON_FIXED, "O") ")") diff --git a/0042-src-common-dout.h.patch b/0042-src-common-dout.h.patch deleted file mode 100644 index 129f50e..0000000 --- a/0042-src-common-dout.h.patch +++ /dev/null @@ -1,141 +0,0 @@ -From 0eace4ea9ea42412d4d6a16d24a8660642e41173 Mon Sep 17 00:00:00 2001 -From: Radoslaw Zarzynski -Date: Wed, 24 Jan 2024 17:22:44 +0000 -Subject: [PATCH] common/dout: fix FTBFS on GCC 14 -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -The following problem has been reported by Kaleb Keithley: - -``` -/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h: In lambda function: -/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:184:73: error: call to non-‘constexpr’ function ‘virtual unsigned int DoutPrefixProvider::get_subsys() const’ - 184 | dout_impl(pdpp->get_cct(), ceph::dout::need_dynamic(pdpp->get_subsys()), v) \ - | ~~~~~~~~~~~~~~~~^~ -/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:155:58: note: in definition of macro ‘dout_impl’ - 155 | return (cctX->_conf->subsys.template should_gather()); \ - | ^~~ -/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h:3617:3: note: in expansion of macro ‘ldpp_dout’ - 3617 | ldpp_dout(dpp, 10) << "build_prior all_probe " << all_probe << dendl; - | ^~~~~~~~~ -``` - -For details of the problem and the idea behind the fix, -please refer to the comment this commit brings to `dout.h`. - -The minimized replicator that the facilitated Goldbot-based -investigation: - -```cpp -namespace ceph::dout { - -template -struct dynamic_marker_t { - T value; - // constexpr ctor isn't needed as it's an aggregate type - constexpr operator T() const { return value; } -}; - -template -constexpr dynamic_marker_t need_dynamic(T&& t) { - return dynamic_marker_t{ std::forward(t) }; -} - -template -struct is_dynamic : public std::false_type {}; - -template -struct is_dynamic> : public std::true_type {}; - -} // ceph::dout - -struct subsys_t { - template - bool should_gather() const { - return true; - } - bool should_gather(const unsigned sub, int level) const { - return false; - } -}; - -static subsys_t subsys; - - do { \ - const bool should_gather = [&](const auto cctX) { \ - if constexpr (ceph::dout::is_dynamic::value || \ - ceph::dout::is_dynamic::value) { \ - std::cout << "the dynamic path" << std::endl; \ - return subsys.should_gather(sub, v); \ - } else { \ - /* The parentheses are **essential** because commas in angle \ - * brackets are NOT ignored on macro expansion! A language's \ - * limitation, sorry. */ \ - std::cout << "the static path" << std::endl; \ - /*return subsys.should_gather(sub, v);*/ \ - return (subsys.template should_gather()); \ - } \ - }(cct); \ - } while (0) - - if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \ - dout_impl(42, sub, v) - - if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \ - dout_impl(42, ceph::dout::need_dynamic(42), v) - -int main() { - std::random_device dev; - std::mt19937 rng(dev()); - std::uniform_int_distribution dist6(1,6); // distribution in range [1, 6] - - int sub = dist6(rng); - ldpp_dout("mocked out", sub); - //ldpp_subdout("mocked out", 4, 3); -} -``` - -Fixes: https://tracker.ceph.com/issues/64050 -Signed-off-by: Radoslaw Zarzynski ---- - src/common/dout.h | 20 +++++++++++++++----- - 1 file changed, 15 insertions(+), 5 deletions(-) - -diff --git a/src/common/dout.h b/src/common/dout.h -index 4cd60efff8fef..6516060c5438e 100644 ---- a/src/common/dout.h -+++ b/src/common/dout.h -@@ -144,17 +144,27 @@ struct is_dynamic> : public std::true_type {}; - #else - #define dout_impl(cct, sub, v) \ - do { \ -- const bool should_gather = [&](const auto cctX) { \ -- if constexpr (ceph::dout::is_dynamic::value || \ -- ceph::dout::is_dynamic::value) { \ -+ const bool should_gather = [&](const auto cctX, auto sub_, auto v_) { \ -+ /* The check is performed on `sub_` and `v_` to leverage the C++'s \ -+ * guarantee on _discarding_ one of blocks of `if constexpr`, which \ -+ * includes also the checks for ill-formed code (`should_gather<>` \ -+ * must not be feed with non-const expresions), BUT ONLY within \ -+ * a template (thus the generic lambda) and under the restriction \ -+ * it's dependant on a parameter of this template). \ -+ * GCC prior to v14 was not enforcing these restrictions. */ \ -+ if constexpr (ceph::dout::is_dynamic::value || \ -+ ceph::dout::is_dynamic::value) { \ - return cctX->_conf->subsys.should_gather(sub, v); \ - } else { \ -+ constexpr auto sub_helper = static_cast(sub); \ -+ constexpr auto v_helper = static_cast(v); \ - /* The parentheses are **essential** because commas in angle \ - * brackets are NOT ignored on macro expansion! A language's \ - * limitation, sorry. */ \ -- return (cctX->_conf->subsys.template should_gather()); \ -+ return (cctX->_conf->subsys.template should_gather()); \ - } \ -- }(cct); \ -+ }(cct, sub, v); \ - \ - if (should_gather) { \ - ceph::logging::MutableEntry _dout_e(v, sub); \ diff --git a/0046-src-s3select-include-s3select_parquet_intrf.h.patch b/0046-src-s3select-include-s3select_parquet_intrf.h.patch deleted file mode 100644 index 39c27f1..0000000 --- a/0046-src-s3select-include-s3select_parquet_intrf.h.patch +++ /dev/null @@ -1,13 +0,0 @@ ---- ceph-19.2.0/src/s3select/include/s3select_parquet_intrf.h.orig 2024-11-26 15:08:27.460400263 -0500 -+++ ceph-19.2.0/src/s3select/include/s3select_parquet_intrf.h 2024-11-26 15:09:22.636480526 -0500 -@@ -1245,8 +1245,8 @@ - } - - file_metadata_ = -- FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_); -- //FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9 -+ //FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_); -+ FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9 - } - - void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter( diff --git a/0056-libarrow-20.0.0.patch b/0056-libarrow-20.0.0.patch index f33897c..d533f82 100644 --- a/0056-libarrow-20.0.0.patch +++ b/0056-libarrow-20.0.0.patch @@ -759,824 +759,9 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h ceph-19 +#else +#include "internal_file_decryptor_20.h" +#endif -diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_functions.h ceph-19.2.2/src/s3select/include/s3select_functions.h ---- ceph-19.2.2/src/s3select.orig/include/s3select_functions.h 2024-03-26 19:23:12.000000000 -0400 -+++ ceph-19.2.2/src/s3select/include/s3select_functions.h 2025-07-07 10:22:00.000000000 -0400 -@@ -505,7 +505,7 @@ - - std::string print(int ident) override - { -- return std::string(0); -+ return std::string{}; - } - - void push_argument(base_statement* arg) -@@ -2268,6 +2268,17 @@ - -- avoid timeout upon long processing #152 - -- replace assert with an exception to avoid crashing the process #151 - -- fix for the use-case of not-operator on a string #153 -+ -+-- the negation-operation may cause a wrong-build of the AST, that later may cause a crash. that operation is missing handling of several operators. #160 -+-- the crash happened upon calling more than once the parse_query, the second call accessed an incomplete object in the AST. #160 -+ -+-- fix for identifies key-value as key-object or key-array #161 -+-- wrong initialization of the JSON parser engine had caused missing a projection-key-path upon using different combinations of the from-clause path. #161 -+-- a fix for copy-constructor, upon a JSON value is an empty string, it causes a wrong result. #161 -+-- modification for JSON star operation, the new-type container saves only unique keys, to avoid high memory consumption. #161 -+-- the from-clause can handle a wild-card. i.e. upon wild-card(*) it skips the corresponding part in projection-key-path. #161 -+-- key-path may include meta-char(like a dot) select _1."i.e."[0] the "i.e." is part of the key-path. #161 -+ - )"; - - _fn_engine_version() -diff -ur ceph-19.2.2/src/s3select.orig/include/s3select.h ceph-19.2.2/src/s3select/include/s3select.h ---- ceph-19.2.2/src/s3select.orig/include/s3select.h 2024-03-26 19:23:12.000000000 -0400 -+++ ceph-19.2.2/src/s3select/include/s3select.h 2025-07-07 10:22:00.000000000 -0400 -@@ -45,6 +45,43 @@ - - static s3select_reserved_word g_s3select_reserve_word;//read-only - -+struct expr_queue { -+ //purpose: a warpper for std::vector for monitoring the expression queue actions. -+ //upon an SQL expression is resolved by the engine it is pushed into the queue for later operations -+ std::vector m_expr_queue; -+ -+ base_statement* back() -+ { -+ //expression queue must contain an element upon accessing it. upon an empty expression queue to throw an exception. -+ if(m_expr_queue.empty()) -+ { -+ throw base_s3select_exception("expression queue is empty, abort parsing.", base_s3select_exception::s3select_exp_en_t::FATAL); -+ } -+ return m_expr_queue.back(); -+ } -+ -+ void push_back(base_statement* bs) -+ { -+ m_expr_queue.push_back(bs); -+ } -+ -+ void pop_back() -+ { -+ return m_expr_queue.pop_back(); -+ } -+ -+ bool empty() -+ { -+ return m_expr_queue.empty(); -+ } -+ -+ void clear() -+ { -+ return m_expr_queue.clear(); -+ } -+ -+}; -+ - struct actionQ - { - // upon parser is accepting a token (lets say some number), -@@ -55,7 +92,7 @@ - std::vector addsubQ; - std::vector arithmetic_compareQ; - std::vector logical_compareQ; -- std::vector exprQ; -+ expr_queue exprQ; - std::vector funcQ; - std::vector whenThenQ; - std::vector inPredicateQ; -@@ -144,6 +181,12 @@ - }; - static push_json_from_clause g_push_json_from_clause; - -+struct push_json_from_clause_key_path : public base_ast_builder -+{ -+ void builder(s3select* self, const char* a, const char* b) const; -+}; -+static push_json_from_clause_key_path g_push_json_from_clause_key_path; -+ - struct push_limit_clause : public base_ast_builder - { - void builder(s3select* self, const char* a, const char* b) const; -@@ -513,6 +556,12 @@ - - int semantic() - { -+ if(get_projections_list().empty()) -+ { -+ error_description = "projections list is empty, AST is wrong,"; -+ throw base_s3select_exception(error_description, base_s3select_exception::s3select_exp_en_t::FATAL); -+ } -+ - for (const auto &e : get_projections_list()) - { - e->resolve_node(); -@@ -746,7 +795,7 @@ - - json_s3_object = ((S3SELECT_KW(JSON_ROOT_OBJECT)) >> *(bsc::str_p(".") >> json_path_element))[BOOST_BIND_ACTION(push_json_from_clause)]; - -- json_path_element = bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_")) ]; -+ json_path_element = (bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_") ) ] | bsc::str_p("*") | (string))[BOOST_BIND_ACTION(push_json_from_clause_key_path)]; - - object_path = "/" >> *( fs_type >> "/") >> fs_type; - -@@ -872,11 +921,12 @@ - - variable = (variable_name >> "." >> variable_name) | variable_name; - -+ // json_variable_name is the JSON projection, i.e. _1.a.b[10] - json_variable_name = bsc::str_p("_1") >> +("." >> (json_array | json_object) ); - -- json_object = (variable_name)[BOOST_BIND_ACTION(push_json_object)]; -+ json_object = (variable_name | string)[BOOST_BIND_ACTION(push_json_object)]; - -- json_array = (variable_name >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)]; -+ json_array = ((variable_name | string) >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)]; - } - - -@@ -936,35 +986,33 @@ - self->getAction()->exprQ.clear(); - } - --void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const -+std::string json_path_remove_double_quote(const char* a, const char* b) -+{ -+ //upon query accessing key which contains meta-char, it must use string-construct(double quotes), -+ //the engine should remove double quotes for later processing. -+ std::string token(a, b); -+ if(*a == '"') //TODO single quote ? -+ { -+ std::string tmp = token.substr(1,token.find('"',1)-1); -+ token = tmp; -+ } -+ return token; -+} -+ -+void push_json_from_clause_key_path::builder(s3select* self, const char* a, const char* b) const - { -- std::string token(a, b),table_name,alias_name; - -- //TODO handle the star-operation ('*') in from-clause. build the parameters for json-reader search-api's. -- std::vector variable_key_path; -- const char* delimiter = "."; -- auto pos = token.find(delimiter); -+ std::string token = json_path_remove_double_quote(a,b); -+ self->getAction()->json_from_clause.push_back(token); -+ -+} - -- if(pos != std::string::npos) -- { -- token = token.substr(strlen(JSON_ROOT_OBJECT)+1,token.size()); -- pos = token.find(delimiter); -- do -- { -- variable_key_path.push_back(token.substr(0,pos)); -- if(pos != std::string::npos) -- token = token.substr(pos+1,token.size()); -- else -- token = ""; -- pos = token.find(delimiter); -- }while(token.size()); -- } -- else -+void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const -+{ -+ if(self->getAction()->json_from_clause.size() == 0) - { -- variable_key_path.push_back(JSON_ROOT_OBJECT); -+ self->getAction()->json_from_clause.push_back(JSON_ROOT_OBJECT); - } -- -- self->getAction()->json_from_clause = variable_key_path; - } - - void push_limit_clause::builder(s3select* self, const char* a, const char* b) const -@@ -1086,7 +1134,6 @@ - {//purpose: handle the use case of json-variable structure (_1.a.b.c) - - std::string token(a, b); -- std::vector variable_key_path; - - //the following flow determine the index per json variable reside on statement. - //per each discovered json_variable, it search the json-variables-vector whether it already exists. -@@ -1116,7 +1163,8 @@ - - void push_json_array_name::builder(s3select* self, const char* a, const char* b) const - { -- std::string token(a, b); -+ std::string token = json_path_remove_double_quote(a,b); -+ - size_t found = token.find("["); - std::string array_name = token.substr(0,found); - -@@ -1143,7 +1191,7 @@ - - void push_json_object::builder(s3select* self, const char* a, const char* b) const - { -- std::string token(a, b); -+ std::string token = json_path_remove_double_quote(a,b); - - //DEBUG - TEMP std::cout << "push_json_object " << token << std::endl; - -@@ -1385,7 +1433,7 @@ - } - else - { -- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL); -+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, expression is missing."), base_s3select_exception::s3select_exp_en_t::FATAL); - } - - //upon NOT operator, the logical and arithmetical operators are "tagged" to negate result. -@@ -1394,7 +1442,8 @@ - logical_operand* f = S3SELECT_NEW(self, logical_operand, pred); - self->getAction()->exprQ.push_back(f); - } -- else if (dynamic_cast<__function*>(pred) || dynamic_cast(pred) || dynamic_cast(pred)) -+ else if (dynamic_cast<__function*>(pred) || dynamic_cast(pred) || dynamic_cast(pred) -+ || dynamic_cast(pred) || dynamic_cast(pred)) - { - negate_function_operation* nf = S3SELECT_NEW(self, negate_function_operation, pred); - self->getAction()->exprQ.push_back(nf); -@@ -1406,7 +1455,7 @@ - } - else - { -- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL); -+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, appropiate operator is missing"), base_s3select_exception::s3select_exp_en_t::FATAL); - } - } - -@@ -2409,6 +2458,7 @@ - } - - result.append(res->to_string()); -+ //TODO the strlen should replace with the size of the string(performance perspective) - m_returned_bytes_size += strlen(res->to_string()); - ++j; - } -@@ -2425,6 +2475,8 @@ - if(m_csv_defintion.output_json_format && projections_resuls.values.size()) { - json_result_format(projections_resuls, result, output_delimiter); - result.append(output_row_delimiter); -+ //TODO to add asneeded -+ //TODO "flush" the result string - return; - } - -@@ -2444,10 +2496,6 @@ - set_processing_time_error(); - } - -- -- if(m_fp_ext_debug_mesg) -- m_fp_ext_debug_mesg(column_result.data()); -- - if (m_csv_defintion.quote_fields_always) { - std::ostringstream quoted_result; - quoted_result << std::quoted(column_result,m_csv_defintion.output_quot_char, m_csv_defintion.escape_char); -@@ -2475,9 +2523,30 @@ - if(!m_aggr_flow) { - result.append(output_row_delimiter); - m_returned_bytes_size += output_delimiter.size(); -- } -+ } -+ -+ -+//TODO config / default-value -+#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024) -+ if(m_fp_s3select_result_format && m_fp_s3select_header_format) -+ { -+ if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT) -+ {//there are systems that might resject the response due to its size. -+ m_fp_s3select_result_format(result); -+ m_fp_s3select_header_format(result); -+ } -+ } - } - -+ void flush_sql_result(std::string& result) -+ {//purpose: flush remaining data reside in the buffer -+ if(m_fp_s3select_result_format && m_fp_s3select_header_format) -+ { -+ m_fp_s3select_result_format(result); -+ m_fp_s3select_header_format(result); -+ } -+ } -+ - Status getMatchRow( std::string& result) - { - multi_values projections_resuls; -@@ -2621,8 +2690,6 @@ - - }; //base_s3object - --//TODO config / default-value --#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024) - class csv_object : public base_s3object - { - -@@ -2935,16 +3002,6 @@ - return -1; - } - } -- -- if(m_fp_s3select_result_format && m_fp_s3select_header_format) -- { -- if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT) -- {//there are systems that might resject the response due to its size. -- m_fp_s3select_result_format(result); -- m_fp_s3select_header_format(result); -- } -- } -- - if (m_sql_processing_status == Status::END_OF_STREAM) - { - break; -@@ -2960,12 +3017,7 @@ - - } while (true); - -- if(m_fp_s3select_result_format && m_fp_s3select_header_format) -- { //note: it may produce empty response(more the once) -- //upon empty result, it should return *only* upon last call. -- m_fp_s3select_result_format(result); -- m_fp_s3select_header_format(result); -- } -+ flush_sql_result(result); - - return 0; - } -@@ -2976,7 +3028,7 @@ - { - - private: -- parquet_file_parser* object_reader; -+ std::unique_ptr object_reader; - parquet_file_parser::column_pos_t m_where_clause_columns; - parquet_file_parser::column_pos_t m_projections_columns; - std::vector m_predicate_values; -@@ -2985,11 +3037,14 @@ - - public: - -- parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query),object_reader(nullptr) -+ class csv_definitions : public s3select_csv_definitions -+ {}; -+ -+ parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query) - { - try{ - -- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr -+ object_reader = std::make_unique(parquet_file_name,rgw); - } catch(std::exception &e) - { - throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL); -@@ -2998,7 +3053,7 @@ - parquet_query_setting(nullptr); - } - -- parquet_object() : base_s3object(nullptr),object_reader(nullptr) -+ parquet_object() : base_s3object(nullptr), not_to_increase_first_time(true) - {} - - void parquet_query_setting(s3select *s3_query) -@@ -3020,13 +3075,7 @@ - } - - ~parquet_object() -- { -- if(object_reader != nullptr) -- { -- delete object_reader; -- } -- -- } -+ {} - - std::string get_error_description() - { -@@ -3038,11 +3087,11 @@ - return m_s3_select != nullptr; - } - -- void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) //TODO duplicate code -+ void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw,csv_definitions parquet) //TODO duplicate code - { - try{ -- -- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr -+ m_csv_defintion = parquet; -+ object_reader = std::make_unique(parquet_file_name,rgw); - } catch(std::exception &e) - { - throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL); -@@ -3081,26 +3130,6 @@ - } - } - --#define S3SELECT_RESPONSE_SIZE_LIMIT (4 * 1024 * 1024) -- if (result.size() > S3SELECT_RESPONSE_SIZE_LIMIT) -- {//AWS-cli limits response size the following callbacks send response upon some threshold -- if(m_fp_s3select_result_format) -- m_fp_s3select_result_format(result); -- -- if (!is_end_of_stream() && (get_sql_processing_status() != Status::LIMIT_REACHED)) -- { -- if(m_fp_s3select_header_format) -- m_fp_s3select_header_format(result); -- } -- } -- else -- { -- if (is_end_of_stream() || (get_sql_processing_status() == Status::LIMIT_REACHED)) -- { -- if(m_fp_s3select_result_format) -- m_fp_s3select_result_format(result); -- } -- } - - //TODO is_end_of_stream() required? - if (get_sql_processing_status() == Status::END_OF_STREAM || is_end_of_stream() || get_sql_processing_status() == Status::LIMIT_REACHED) -@@ -3110,6 +3139,7 @@ - - } while (1); - -+ flush_sql_result(result); - return 0; - } - -@@ -3181,10 +3211,6 @@ - f_push_key_value_into_scratch_area_per_star_operation = [this](s3selectEngine::scratch_area::json_key_value_t& key_value) - {return push_key_value_into_scratch_area_per_star_operation(key_value);}; - -- //setting the container for all json-variables, to be extracted by the json reader -- JsonHandler.set_statement_json_variables(query->get_json_variables_access()); -- -- - //calling to getMatchRow. processing a single row per each call. - JsonHandler.set_s3select_processing_callback(f_sql); - //upon excat match between input-json-key-path and sql-statement-variable-path the callback pushes to scratch area -@@ -3226,6 +3252,9 @@ - } - - m_sa->set_parquet_type();//TODO json type -+ -+ //setting the container for all json-variables, to be extracted by the json reader -+ JsonHandler.set_statement_json_variables(query->get_json_variables_access()); - } - - json_object(s3select* query):base_s3object(query),m_processed_bytes(0),m_end_of_stream(false),m_row_count(0),star_operation_ind(false),m_init_json_processor_ind(false) -@@ -3293,7 +3322,10 @@ - - int push_key_value_into_scratch_area_per_star_operation(s3selectEngine::scratch_area::json_key_value_t& key_value) - { -- m_sa->get_star_operation_cont()->push_back( key_value ); -+ //upon star-operation on nested JSON, there could be many keys in a single row (actually, there is no limitation). -+ //for many cases these keys are duplicated in the scope of a single-row (row is defined according to SQL statement). -+ //the following routine saves only unique keys. -+ m_sa->json_push_key_value_per_star_operation(key_value); - return 0; - } - -diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h ceph-19.2.2/src/s3select/include/s3select_json_parser.h ---- ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h 2024-03-26 19:23:12.000000000 -0400 -+++ ceph-19.2.2/src/s3select/include/s3select_json_parser.h 2025-07-07 10:22:00.000000000 -0400 -@@ -340,12 +340,12 @@ - - if(reader_position_state().required_path.size())//current state is a key - { -- std::vector* filter = &reader_position_state().required_path; -+ std::vector* projection_key_path = &reader_position_state().required_path; - auto required_key_depth_size = reader_position_state().required_key_depth_size; - if(std::equal((*key_path).begin()+(*from_clause).size() + required_key_depth_size, //key-path-start-point + from-clause-depth-size + key-depth - (*key_path).end(), -- (*filter).begin(), -- (*filter).end(), iequal_predicate)) -+ (*projection_key_path).begin(), -+ (*projection_key_path).end(), iequal_predicate)) - { - increase_current_state();//key match according to user request, advancing to the next state - } -@@ -640,15 +640,45 @@ - m_current_depth_non_anonymous++; - } - -- if(from_clause.size() == 0 || std::equal(key_path.begin(), key_path.end(), from_clause.begin(), from_clause.end(), iequal_predicate)) { -- prefix_match = true; -- } -- - variable_match_operations.key(); - - return true; - } - -+ template -+ bool from_clause_matcher(std::vector& _key_path, -+ std::vector& _from_clause, -+ predicate p) -+ { -+ //iterate on both path's -+ //upon a part of from-clause is '*' it consider 'equal' to the counter part (projection), should skip to the next part -+ // -+ //from-clause = a.*.c ; projection-key = a.b.c ; since the '*' is on the secod position -+ //it means b=exists-in-from-clause, it should skip to next part (the third). -+ -+ std::vector::iterator it_key_path = _key_path.begin(); -+ std::vector::iterator it_from_clause = _from_clause.begin(); -+ -+ while(it_from_clause != _from_clause.end()) -+ { -+ if (it_key_path == _key_path.end()) return false; -+ -+ if ((it_from_clause->compare("*")==0) || p(*it_key_path,*it_from_clause)) -+ {it_key_path++; it_from_clause++;} -+ else -+ return false; -+ }; -+ return true; -+ -+ }; -+ -+ -+ void set_prefix_match(){ -+ if(from_clause.size() == 0 || from_clause_matcher(key_path, from_clause, iequal_predicate)) { -+ prefix_match = true; //it is not prefix_match in the case its a key/value . it is a prefix match in the case it is a key of array or key of an object -+ } -+ } -+ - bool is_already_row_started() - { - if(state == row_state::OBJECT_START_ROW || state == row_state::ARRAY_START_ROW) -@@ -658,6 +688,8 @@ - } - - bool StartObject() { -+ set_prefix_match(); -+ - json_element_state.push_back(OBJECT_STATE); - m_current_depth++; - if(key_path.size()){ -@@ -690,6 +722,8 @@ - } - - bool StartArray() { -+ set_prefix_match(); -+ - json_element_state.push_back(ARRAY_STATE); - m_current_depth++; - if(key_path.size()){ -diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_oper.h ceph-19.2.2/src/s3select/include/s3select_oper.h ---- ceph-19.2.2/src/s3select.orig/include/s3select_oper.h 2024-03-26 19:23:12.000000000 -0400 -+++ ceph-19.2.2/src/s3select/include/s3select_oper.h 2025-07-07 10:22:00.000000000 -0400 -@@ -592,9 +592,9 @@ - - value_En_t _type() const { return type; } - -- void set_json_key_path(std::vector& key_path) -+ void set_json_key_path(const std::vector& key_path) - { -- m_json_key = key_path; -+ m_json_key = key_path;//TODO not efficient - } - - const char* to_string() //TODO very intensive , must improve this -@@ -693,9 +693,10 @@ - m_str_value = o.m_str_value; - __val.str = m_str_value.data(); - } -- else if(o.__val.str) -+ else if(o.__val.str)//it is done upon using the set_string_nocopy - { -- __val.str = o.__val.str; -+ m_str_value.assign(o.__val.str);//need to create a copy for this -+ __val.str = m_str_value.data(); - } - } - else -@@ -717,9 +718,10 @@ - m_str_value = o.m_str_value; - __val.str = m_str_value.data(); - } -- else if(o.__val.str) -+ else if(o.__val.str)//it is done upon using the set_string_nocopy - { -- __val.str = o.__val.str; -+ m_str_value.assign(o.__val.str); -+ __val.str = m_str_value.data(); - } - } - else -@@ -1137,6 +1139,54 @@ - } - } - -+///////////////////////// -+#include -+class json_key_value_store -+{ -+//hash function for std::vector -+struct VectorStringHash { -+ std::size_t operator()(const std::vector& vec) const { -+ std::size_t seed = 0; -+ for (const auto& str : vec) { -+ for (char c : str) { -+ // boost::hash_combine_size_t -+ seed ^= std::hash()(c) + 0x9e3779b9 + (seed << 6) + (seed >> 2); -+ } -+ } -+ return seed; -+ } -+}; -+ -+//equality comparison function for std::vector -+struct VectorStringEqual { -+ bool operator()(const std::vector& lhs, const std::vector& rhs) const { -+ return lhs == rhs; -+ } -+}; -+ -+public: -+ -+//NOTE: using unordered_map(and also std::map) means that output result is NOT as the input order (TODO consider using an order column) -+//order means, order of keys, as appear in the input. order has no affect on the correctness of results. -+//the current flow, for identicals keys, save the first key/value and ignore all the rest. -+// -+//this type of container avoid huge memeory consumption upon a very big row(e.g. { "root" : [ long-and-nested-objects ] } ) -+//a 'select * from s3object[*];' has to scan the entire "root" object before returing a result, -+//the star-operation must upload all keys-values for that row. -+//without a unique-key machanism it will consume large amount of memory(entire input JSON object). -+// -+//NOTE: upon huge JSON object containing unique-keys(random names), it will cause high memory consumption. -+//TODO:consider using container size limitation. -+typedef std::unordered_map, value, VectorStringHash, VectorStringEqual> _unorder_map_t; -+ -+_unorder_map_t json_key_value_store; -+ -+void insert(const std::vector& key,value v) -+{ -+ json_key_value_store.emplace( std::pair,value>(key,v) ); -+} -+ -+}; - - class scratch_area - { -@@ -1155,8 +1205,8 @@ - public: - - typedef std::pair,value> json_key_value_t; -- typedef std::vector< json_key_value_t > json_star_op_cont_t; -- json_star_op_cont_t m_json_star_operation; -+ -+ json_key_value_store m_json_star_operation; - - scratch_area():m_upper_bound(-1),parquet_type(false),buff_loc(0),max_json_idx(-1) - { -@@ -1168,14 +1218,20 @@ - delete m_schema_values; - } - -- json_star_op_cont_t* get_star_operation_cont() -+ json_key_value_store::_unorder_map_t* get_star_operation_cont() -+ { -+ return &m_json_star_operation.json_key_value_store; -+ } -+ -+ void json_push_key_value_per_star_operation(json_key_value_t& key_value) - { -- return &m_json_star_operation; -+ //identical keys(and thier values) are not saved. -+ m_json_star_operation.insert(key_value.first,key_value.second); - } - - void clear_data() - { -- m_json_star_operation.clear(); -+ m_json_star_operation.json_key_value_store.clear(); - for(int i=0;i<=max_json_idx;i++) - { - (*m_schema_values)[i].setnull(); -@@ -1857,8 +1913,8 @@ - else - { - m_scratch->get_column_value(column_pos,var_value); -- //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL -- if (var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){ -+ //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL(CSV data source) -+ if (!is_json_statement() && var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){ - var_value.setnull();//TODO is it correct for Parquet - } - } diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h --- ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h 2024-03-26 19:23:12.000000000 -0400 +++ ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h 2025-07-07 13:08:00.747275733 -0400 -@@ -434,7 +434,7 @@ - #if ARROW_VERSION_MAJOR < 9 - fd_ = -1; - #else -- fd_.Close(); -+ return fd_.Close(); - #endif - //RETURN_NOT_OK(::arrow::internal::FileClose(fd)); - } -@@ -589,17 +589,12 @@ - public: - - ~ReadableFileImpl() -- { -- if(IMPL != nullptr) -- { -- delete IMPL; -- } -- } -+ {} - - #ifdef CEPH_USE_FS -- explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=new OSFile();} -+ explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=std::make_unique();} - #endif -- explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=new RGWimpl(rgw);} -+ explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=std::make_unique(rgw);} - - Status Open(const std::string& path) { return IMPL->OpenReadable(path); } - -@@ -649,7 +644,7 @@ - return Status::OK(); - } - -- ObjectInterface *IMPL;//TODO to declare in ObjectInterface -+ std::unique_ptr IMPL; - - private: - -@@ -752,6 +747,30 @@ - std::unique_ptr contents_; - }; - -+#define RGW_default_buffer_size 1024*1024*1024 -+class S3select_Config { -+public: -+ static S3select_Config& getInstance() { -+ static S3select_Config instance; -+ return instance; -+ } -+ -+ void set_s3select_reader_properties(uint64_t value) { this->m_reader_properties = value; } -+ uint64_t get_s3select_reader_properties() const { return m_reader_properties; } -+ -+private: -+ uint64_t m_reader_properties; -+ S3select_Config() : m_reader_properties(RGW_default_buffer_size) {} // Private constructor -+}; -+ -+ReaderProperties s3select_reader_properties() { -+ -+ static ReaderProperties default_reader_properties; -+ default_reader_properties.enable_buffered_stream(); -+ default_reader_properties.set_buffer_size(S3select_Config::getInstance().get_s3select_reader_properties()); -+ return default_reader_properties; -+} -+ - class PARQUET_EXPORT ParquetFileReader { - public: - // Declare a virtual class 'Contents' to aid dependency injection and more -@@ -760,7 +779,7 @@ - struct PARQUET_EXPORT Contents { - static std::unique_ptr Open( - std::shared_ptr<::arrow::io::RandomAccessFile> source, -- const ReaderProperties& props = default_reader_properties(), -+ const ReaderProperties& props = s3select_reader_properties(), - std::shared_ptr metadata = NULLPTR); - - virtual ~Contents() = default; -@@ -781,21 +800,21 @@ - ARROW_DEPRECATED("Use arrow::io::RandomAccessFile version") - static std::unique_ptr Open( - std::unique_ptr source, -- const ReaderProperties& props = default_reader_properties(), -+ const ReaderProperties& props = s3select_reader_properties(), - std::shared_ptr metadata = NULLPTR); - - // Create a file reader instance from an Arrow file object. Thread-safety is - // the responsibility of the file implementation - static std::unique_ptr Open( - std::shared_ptr<::arrow::io::RandomAccessFile> source, -- const ReaderProperties& props = default_reader_properties(), -+ const ReaderProperties& props = s3select_reader_properties(), - std::shared_ptr metadata = NULLPTR); - - // API Convenience to open a serialized Parquet file on disk, using Arrow IO - // interfaces. - static std::unique_ptr OpenFile( - const std::string& path,s3selectEngine::rgw_s3select_api* rgw, bool memory_map = true, -- const ReaderProperties& props = default_reader_properties(), -+ const ReaderProperties& props = s3select_reader_properties(), - std::shared_ptr metadata = NULLPTR - ); - @@ -983,6 +1002,7 @@ throw ParquetException("Encrypted files cannot contain more than 32767 row groups"); } @@ -1611,15 +796,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19. } private: -@@ -1039,7 +1078,7 @@ - class SerializedFile : public ParquetFileReader::Contents { - public: - SerializedFile(std::shared_ptr source, -- const ReaderProperties& props = default_reader_properties()) -+ const ReaderProperties& props = s3select_reader_properties()) - : source_(std::move(source)), properties_(props) { - PARQUET_ASSIGN_OR_THROW(source_size_, source_->GetSize()); - } @@ -1052,7 +1091,9 @@ } @@ -1648,21 +824,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19. int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len; uint32_t metadata_len = footer_len - crypto_metadata_len; -@@ -1244,9 +1293,13 @@ - std::to_string(metadata_buffer->size()) + " bytes)"); - } - -+#if ARROW_VERSION_MAJOR > 9 -+ file_metadata_ = -+ FileMetaData::Make(metadata_buffer->data(), &metadata_len, s3select_reader_properties(), file_decryptor_); -+#else - file_metadata_ = - FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_); -- //FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9 -+#endif - } - - void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter( @@ -1259,9 +1312,18 @@ EncryptionAlgorithm algo = file_metadata_->encryption_algorithm(); // Handle AAD prefix @@ -1682,57 +843,3 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19. // set the InternalFileDecryptor in the metadata as well, as it's used // for signature verification and for ColumnChunkMetaData creation. #if GAL_set_file_decryptor_declare_private -@@ -1584,7 +1646,7 @@ - int m_num_row_groups; - std::shared_ptr m_file_metadata; - std::unique_ptr m_parquet_reader; -- std::vector m_column_readers; -+ std::vector> m_column_readers; - s3selectEngine::rgw_s3select_api* m_rgw_s3select_api; - - public: -@@ -1603,12 +1665,7 @@ - } - - ~parquet_file_parser() -- { -- for(auto r : m_column_readers) -- { -- delete r; -- } -- } -+ {} - - int load_meta_data() - { -@@ -1658,7 +1715,7 @@ - } - } - -- m_column_readers.push_back(new column_reader_wrap(m_parquet_reader,i)); -+ m_column_readers.push_back(std::make_shared(m_parquet_reader,i)); - } - - return 0; ---- ceph-19.2.2/src/rgw/rgw_s3select.cc.orig 2025-07-07 14:24:23.371796969 -0400 -+++ ceph-19.2.2/src/rgw/rgw_s3select.cc 2025-07-07 17:45:30.655554945 -0400 -@@ -472,6 +472,7 @@ - if (!m_s3_parquet_object.is_set()) { - //parsing the SQL statement. - s3select_syntax.parse_query(m_sql_query.c_str()); -+ parquet_object::csv_definitions parquet; - - m_s3_parquet_object.set_external_system_functions(fp_s3select_continue, - fp_s3select_result_format, -@@ -479,8 +480,9 @@ - fp_debug_mesg); - - try { -- //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object -- m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api); -+ //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object -+ // m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api); -+ m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api, parquet); - } catch(base_s3select_exception& e) { - ldpp_dout(this, 10) << "S3select: failed upon parquet-reader construction: " << e.what() << dendl; - fp_result_header_format(m_aws_response_handler.get_sql_result()); diff --git a/ceph.spec b/ceph.spec index cbdb035..fd3c86f 100644 --- a/ceph.spec +++ b/ceph.spec @@ -217,8 +217,8 @@ # main package definition ################################################################################# Name: ceph -Version: 19.2.2 -Release: 2%{?dist} +Version: 19.2.3 +Release: 1%{?dist} %if 0%{?fedora} || 0%{?rhel} Epoch: 2 %endif @@ -328,6 +328,7 @@ BuildRequires: gperftools-devel >= 2.4 BuildRequires: libaio-devel BuildRequires: libblkid-devel >= 2.17 BuildRequires: cryptsetup-devel +BuildRequires: libnbd-devel BuildRequires: libcurl-devel BuildRequires: libcap-devel BuildRequires: libcap-ng-devel @@ -730,6 +731,7 @@ Requires: python%{python3_pkgversion}-setuptools %if 0%{?fedora} || 0%{?rhel} >= 9 Requires: python%{python3_pkgversion}-grpcio Requires: python%{python3_pkgversion}-grpcio-tools +Requires: python%{python3_pkgversion}-xmltodict %endif %if 0%{?fedora} || 0%{?rhel} Requires: python%{python3_pkgversion}-cherrypy @@ -1784,6 +1786,9 @@ rm -f %{buildroot}%{_mandir}/man8/ceph-bluestore-tool.8* rm -f %{buildroot}%{_unitdir}/ceph-osd@.service rm -f %{buildroot}%{_unitdir}/ceph-osd.target +# Remove ceph-exporter +rm -f %{buildroot}%{_unitdir}/ceph-exporter.service + # Remove rbd-mirror rm -f %{buildroot}%{_bindir}/rbd-mirror rm -f %{buildroot}%{_mandir}/man8/rbd-mirror.8* @@ -2007,6 +2012,7 @@ exit 0 %{_mandir}/man8/rbd-replay-prep.8* %if 0%{with radosgw} %{_mandir}/man8/rgw-orphan-list.8* +%{_mandir}/man8/rgw-gap-list.8* %{_mandir}/man8/rgw-restore-bucket-index.8* %endif %dir %{_datadir}/ceph/ @@ -2343,6 +2349,7 @@ fi %files -n ceph-exporter %{_bindir}/ceph-exporter +%{_unitdir}/ceph-exporter.service %files -n rbd-fuse %{_bindir}/rbd-fuse @@ -2939,6 +2946,9 @@ exit 0 %endif %changelog +* Mon Jul 28 2025 Kaleb S. KEITHLEY - 2:19.2.3-1 +- ceph-19.2.3 GA + * Tue Jul 15 2025 Kaleb S. KEITHLEY - 2:19.2.2-2 - ceph-19.2.2, rebuild with libarrow-20 and liborc-2.1.3 diff --git a/sources b/sources index 15cc338..c4baa17 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (ceph-19.2.2.tar.gz) = ee47c1cb7cb5084b87bcc5a35b3df88fb49683524bba8f2e1ced9d2f8891af53e4b5fb5aa153ed6bce31683625d9bf5176bab9f55bc71671f0e34667948f7285 +SHA512 (ceph-19.2.3.tar.gz) = 278101d2df7bed5363b20c2b065d7a7b26252c8164511257e213ffaa58d509015558183de10bc9281bcbe4d9f85244bcac5bba4db9823e28df6a96d0b687d00a