ceph 19.2.3 GA

This commit is contained in:
Kaleb S. KEITHLEY
2025-07-29 09:09:39 -04:00
parent 23b70803bb
commit 802f8f1577
7 changed files with 13 additions and 1072 deletions
-11
View File
@@ -1,11 +0,0 @@
--- ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h.orig 2022-02-11 17:21:40.268627997 -0500
+++ ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h 2022-02-11 17:21:57.155325437 -0500
@@ -466,7 +466,7 @@
std::string print(int ident) override
{
- return std::string(0);
+ return std::string("");
}
void push_argument(base_statement* arg)
@@ -1,11 +0,0 @@
--- ceph-18.2.3/src/boost/boost/python/call_method.hpp.orig 2024-06-07 11:14:58.579734749 -0400
+++ ceph-18.2.3/src/boost/boost/python/call_method.hpp 2024-06-07 11:19:06.754520504 -0400
@@ -59,7 +59,7 @@
)
{
PyObject* const result =
- PyEval_CallMethod(
+ PyObject_CallMethod(
self
, const_cast<char*>(name)
, const_cast<char*>("(" BOOST_PP_REPEAT_1ST(N, BOOST_PYTHON_FIXED, "O") ")")
-141
View File
@@ -1,141 +0,0 @@
From 0eace4ea9ea42412d4d6a16d24a8660642e41173 Mon Sep 17 00:00:00 2001
From: Radoslaw Zarzynski <rzarzyns@redhat.com>
Date: Wed, 24 Jan 2024 17:22:44 +0000
Subject: [PATCH] common/dout: fix FTBFS on GCC 14
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The following problem has been reported by Kaleb Keithley:
```
/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h: In lambda function:
/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:184:73: error: call to non-‘constexpr’ function ‘virtual unsigned int DoutPrefixProvider::get_subsys() const’
184 | dout_impl(pdpp->get_cct(), ceph::dout::need_dynamic(pdpp->get_subsys()), v) \
| ~~~~~~~~~~~~~~~~^~
/builddir/build/BUILD/ceph-18.2.1/src/common/dout.h:155:58: note: in definition of macro ‘dout_impl’
155 | return (cctX->_conf->subsys.template should_gather<sub, v>()); \
| ^~~
/builddir/build/BUILD/ceph-18.2.1/src/osd/osd_types.h:3617:3: note: in expansion of macro ‘ldpp_dout’
3617 | ldpp_dout(dpp, 10) << "build_prior all_probe " << all_probe << dendl;
| ^~~~~~~~~
```
For details of the problem and the idea behind the fix,
please refer to the comment this commit brings to `dout.h`.
The minimized replicator that the facilitated Goldbot-based
investigation:
```cpp
namespace ceph::dout {
template<typename T>
struct dynamic_marker_t {
T value;
// constexpr ctor isn't needed as it's an aggregate type
constexpr operator T() const { return value; }
};
template<typename T>
constexpr dynamic_marker_t<T> need_dynamic(T&& t) {
return dynamic_marker_t<T>{ std::forward<T>(t) };
}
template<typename T>
struct is_dynamic : public std::false_type {};
template<typename T>
struct is_dynamic<dynamic_marker_t<T>> : public std::true_type {};
} // ceph::dout
struct subsys_t {
template <unsigned SubV, int LvlV>
bool should_gather() const {
return true;
}
bool should_gather(const unsigned sub, int level) const {
return false;
}
};
static subsys_t subsys;
do { \
const bool should_gather = [&](const auto cctX) { \
if constexpr (ceph::dout::is_dynamic<decltype(sub)>::value || \
ceph::dout::is_dynamic<decltype(v)>::value) { \
std::cout << "the dynamic path" << std::endl; \
return subsys.should_gather(sub, v); \
} else { \
/* The parentheses are **essential** because commas in angle \
* brackets are NOT ignored on macro expansion! A language's \
* limitation, sorry. */ \
std::cout << "the static path" << std::endl; \
/*return subsys.should_gather(sub, v);*/ \
return (subsys.template should_gather<sub, v>()); \
} \
}(cct); \
} while (0)
if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \
dout_impl(42, sub, v)
if (decltype(auto) pdpp = (dpp); pdpp) /* workaround -Wnonnull-compare for 'this' */ \
dout_impl(42, ceph::dout::need_dynamic(42), v)
int main() {
std::random_device dev;
std::mt19937 rng(dev());
std::uniform_int_distribution<std::mt19937::result_type> dist6(1,6); // distribution in range [1, 6]
int sub = dist6(rng);
ldpp_dout("mocked out", sub);
//ldpp_subdout("mocked out", 4, 3);
}
```
Fixes: https://tracker.ceph.com/issues/64050
Signed-off-by: Radoslaw Zarzynski <rzarzyns@redhat.com>
---
src/common/dout.h | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/src/common/dout.h b/src/common/dout.h
index 4cd60efff8fef..6516060c5438e 100644
--- a/src/common/dout.h
+++ b/src/common/dout.h
@@ -144,17 +144,27 @@ struct is_dynamic<dynamic_marker_t<T>> : public std::true_type {};
#else
#define dout_impl(cct, sub, v) \
do { \
- const bool should_gather = [&](const auto cctX) { \
- if constexpr (ceph::dout::is_dynamic<decltype(sub)>::value || \
- ceph::dout::is_dynamic<decltype(v)>::value) { \
+ const bool should_gather = [&](const auto cctX, auto sub_, auto v_) { \
+ /* The check is performed on `sub_` and `v_` to leverage the C++'s \
+ * guarantee on _discarding_ one of blocks of `if constexpr`, which \
+ * includes also the checks for ill-formed code (`should_gather<>` \
+ * must not be feed with non-const expresions), BUT ONLY within \
+ * a template (thus the generic lambda) and under the restriction \
+ * it's dependant on a parameter of this template). \
+ * GCC prior to v14 was not enforcing these restrictions. */ \
+ if constexpr (ceph::dout::is_dynamic<decltype(sub_)>::value || \
+ ceph::dout::is_dynamic<decltype(v_)>::value) { \
return cctX->_conf->subsys.should_gather(sub, v); \
} else { \
+ constexpr auto sub_helper = static_cast<decltype(sub_)>(sub); \
+ constexpr auto v_helper = static_cast<decltype(v_)>(v); \
/* The parentheses are **essential** because commas in angle \
* brackets are NOT ignored on macro expansion! A language's \
* limitation, sorry. */ \
- return (cctX->_conf->subsys.template should_gather<sub, v>()); \
+ return (cctX->_conf->subsys.template should_gather<sub_helper, \
+ v_helper>()); \
} \
- }(cct); \
+ }(cct, sub, v); \
\
if (should_gather) { \
ceph::logging::MutableEntry _dout_e(v, sub); \
@@ -1,13 +0,0 @@
--- ceph-19.2.0/src/s3select/include/s3select_parquet_intrf.h.orig 2024-11-26 15:08:27.460400263 -0500
+++ ceph-19.2.0/src/s3select/include/s3select_parquet_intrf.h 2024-11-26 15:09:22.636480526 -0500
@@ -1245,8 +1245,8 @@
}
file_metadata_ =
- FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_);
- //FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9
+ //FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_);
+ FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9
}
void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter(
-893
View File
@@ -759,824 +759,9 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h ceph-19
+#else
+#include "internal_file_decryptor_20.h"
+#endif
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_functions.h ceph-19.2.2/src/s3select/include/s3select_functions.h
--- ceph-19.2.2/src/s3select.orig/include/s3select_functions.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select_functions.h 2025-07-07 10:22:00.000000000 -0400
@@ -505,7 +505,7 @@
std::string print(int ident) override
{
- return std::string(0);
+ return std::string{};
}
void push_argument(base_statement* arg)
@@ -2268,6 +2268,17 @@
-- avoid timeout upon long processing #152
-- replace assert with an exception to avoid crashing the process #151
-- fix for the use-case of not-operator on a string #153
+
+-- the negation-operation may cause a wrong-build of the AST, that later may cause a crash. that operation is missing handling of several operators. #160
+-- the crash happened upon calling more than once the parse_query, the second call accessed an incomplete object in the AST. #160
+
+-- fix for identifies key-value as key-object or key-array #161
+-- wrong initialization of the JSON parser engine had caused missing a projection-key-path upon using different combinations of the from-clause path. #161
+-- a fix for copy-constructor, upon a JSON value is an empty string, it causes a wrong result. #161
+-- modification for JSON star operation, the new-type container saves only unique keys, to avoid high memory consumption. #161
+-- the from-clause can handle a wild-card. i.e. upon wild-card(*) it skips the corresponding part in projection-key-path. #161
+-- key-path may include meta-char(like a dot) select _1."i.e."[0] the "i.e." is part of the key-path. #161
+
)";
_fn_engine_version()
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select.h ceph-19.2.2/src/s3select/include/s3select.h
--- ceph-19.2.2/src/s3select.orig/include/s3select.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select.h 2025-07-07 10:22:00.000000000 -0400
@@ -45,6 +45,43 @@
static s3select_reserved_word g_s3select_reserve_word;//read-only
+struct expr_queue {
+ //purpose: a warpper for std::vector for monitoring the expression queue actions.
+ //upon an SQL expression is resolved by the engine it is pushed into the queue for later operations
+ std::vector<base_statement*> m_expr_queue;
+
+ base_statement* back()
+ {
+ //expression queue must contain an element upon accessing it. upon an empty expression queue to throw an exception.
+ if(m_expr_queue.empty())
+ {
+ throw base_s3select_exception("expression queue is empty, abort parsing.", base_s3select_exception::s3select_exp_en_t::FATAL);
+ }
+ return m_expr_queue.back();
+ }
+
+ void push_back(base_statement* bs)
+ {
+ m_expr_queue.push_back(bs);
+ }
+
+ void pop_back()
+ {
+ return m_expr_queue.pop_back();
+ }
+
+ bool empty()
+ {
+ return m_expr_queue.empty();
+ }
+
+ void clear()
+ {
+ return m_expr_queue.clear();
+ }
+
+};
+
struct actionQ
{
// upon parser is accepting a token (lets say some number),
@@ -55,7 +92,7 @@
std::vector<addsub_operation::addsub_op_t> addsubQ;
std::vector<arithmetic_operand::cmp_t> arithmetic_compareQ;
std::vector<logical_operand::oplog_t> logical_compareQ;
- std::vector<base_statement*> exprQ;
+ expr_queue exprQ;
std::vector<base_statement*> funcQ;
std::vector<base_statement*> whenThenQ;
std::vector<base_statement*> inPredicateQ;
@@ -144,6 +181,12 @@
};
static push_json_from_clause g_push_json_from_clause;
+struct push_json_from_clause_key_path : public base_ast_builder
+{
+ void builder(s3select* self, const char* a, const char* b) const;
+};
+static push_json_from_clause_key_path g_push_json_from_clause_key_path;
+
struct push_limit_clause : public base_ast_builder
{
void builder(s3select* self, const char* a, const char* b) const;
@@ -513,6 +556,12 @@
int semantic()
{
+ if(get_projections_list().empty())
+ {
+ error_description = "projections list is empty, AST is wrong,";
+ throw base_s3select_exception(error_description, base_s3select_exception::s3select_exp_en_t::FATAL);
+ }
+
for (const auto &e : get_projections_list())
{
e->resolve_node();
@@ -746,7 +795,7 @@
json_s3_object = ((S3SELECT_KW(JSON_ROOT_OBJECT)) >> *(bsc::str_p(".") >> json_path_element))[BOOST_BIND_ACTION(push_json_from_clause)];
- json_path_element = bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_")) ];
+ json_path_element = (bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_") ) ] | bsc::str_p("*") | (string))[BOOST_BIND_ACTION(push_json_from_clause_key_path)];
object_path = "/" >> *( fs_type >> "/") >> fs_type;
@@ -872,11 +921,12 @@
variable = (variable_name >> "." >> variable_name) | variable_name;
+ // json_variable_name is the JSON projection, i.e. _1.a.b[10]
json_variable_name = bsc::str_p("_1") >> +("." >> (json_array | json_object) );
- json_object = (variable_name)[BOOST_BIND_ACTION(push_json_object)];
+ json_object = (variable_name | string)[BOOST_BIND_ACTION(push_json_object)];
- json_array = (variable_name >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)];
+ json_array = ((variable_name | string) >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)];
}
@@ -936,35 +986,33 @@
self->getAction()->exprQ.clear();
}
-void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const
+std::string json_path_remove_double_quote(const char* a, const char* b)
+{
+ //upon query accessing key which contains meta-char, it must use string-construct(double quotes),
+ //the engine should remove double quotes for later processing.
+ std::string token(a, b);
+ if(*a == '"') //TODO single quote ?
+ {
+ std::string tmp = token.substr(1,token.find('"',1)-1);
+ token = tmp;
+ }
+ return token;
+}
+
+void push_json_from_clause_key_path::builder(s3select* self, const char* a, const char* b) const
{
- std::string token(a, b),table_name,alias_name;
- //TODO handle the star-operation ('*') in from-clause. build the parameters for json-reader search-api's.
- std::vector<std::string> variable_key_path;
- const char* delimiter = ".";
- auto pos = token.find(delimiter);
+ std::string token = json_path_remove_double_quote(a,b);
+ self->getAction()->json_from_clause.push_back(token);
+
+}
- if(pos != std::string::npos)
- {
- token = token.substr(strlen(JSON_ROOT_OBJECT)+1,token.size());
- pos = token.find(delimiter);
- do
- {
- variable_key_path.push_back(token.substr(0,pos));
- if(pos != std::string::npos)
- token = token.substr(pos+1,token.size());
- else
- token = "";
- pos = token.find(delimiter);
- }while(token.size());
- }
- else
+void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const
+{
+ if(self->getAction()->json_from_clause.size() == 0)
{
- variable_key_path.push_back(JSON_ROOT_OBJECT);
+ self->getAction()->json_from_clause.push_back(JSON_ROOT_OBJECT);
}
-
- self->getAction()->json_from_clause = variable_key_path;
}
void push_limit_clause::builder(s3select* self, const char* a, const char* b) const
@@ -1086,7 +1134,6 @@
{//purpose: handle the use case of json-variable structure (_1.a.b.c)
std::string token(a, b);
- std::vector<std::string> variable_key_path;
//the following flow determine the index per json variable reside on statement.
//per each discovered json_variable, it search the json-variables-vector whether it already exists.
@@ -1116,7 +1163,8 @@
void push_json_array_name::builder(s3select* self, const char* a, const char* b) const
{
- std::string token(a, b);
+ std::string token = json_path_remove_double_quote(a,b);
+
size_t found = token.find("[");
std::string array_name = token.substr(0,found);
@@ -1143,7 +1191,7 @@
void push_json_object::builder(s3select* self, const char* a, const char* b) const
{
- std::string token(a, b);
+ std::string token = json_path_remove_double_quote(a,b);
//DEBUG - TEMP std::cout << "push_json_object " << token << std::endl;
@@ -1385,7 +1433,7 @@
}
else
{
- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL);
+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, expression is missing."), base_s3select_exception::s3select_exp_en_t::FATAL);
}
//upon NOT operator, the logical and arithmetical operators are "tagged" to negate result.
@@ -1394,7 +1442,8 @@
logical_operand* f = S3SELECT_NEW(self, logical_operand, pred);
self->getAction()->exprQ.push_back(f);
}
- else if (dynamic_cast<__function*>(pred) || dynamic_cast<negate_function_operation*>(pred) || dynamic_cast<variable*>(pred))
+ else if (dynamic_cast<__function*>(pred) || dynamic_cast<negate_function_operation*>(pred) || dynamic_cast<variable*>(pred)
+ || dynamic_cast<addsub_operation*>(pred) || dynamic_cast<mulldiv_operation*>(pred))
{
negate_function_operation* nf = S3SELECT_NEW(self, negate_function_operation, pred);
self->getAction()->exprQ.push_back(nf);
@@ -1406,7 +1455,7 @@
}
else
{
- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL);
+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, appropiate operator is missing"), base_s3select_exception::s3select_exp_en_t::FATAL);
}
}
@@ -2409,6 +2458,7 @@
}
result.append(res->to_string());
+ //TODO the strlen should replace with the size of the string(performance perspective)
m_returned_bytes_size += strlen(res->to_string());
++j;
}
@@ -2425,6 +2475,8 @@
if(m_csv_defintion.output_json_format && projections_resuls.values.size()) {
json_result_format(projections_resuls, result, output_delimiter);
result.append(output_row_delimiter);
+ //TODO to add asneeded
+ //TODO "flush" the result string
return;
}
@@ -2444,10 +2496,6 @@
set_processing_time_error();
}
-
- if(m_fp_ext_debug_mesg)
- m_fp_ext_debug_mesg(column_result.data());
-
if (m_csv_defintion.quote_fields_always) {
std::ostringstream quoted_result;
quoted_result << std::quoted(column_result,m_csv_defintion.output_quot_char, m_csv_defintion.escape_char);
@@ -2475,9 +2523,30 @@
if(!m_aggr_flow) {
result.append(output_row_delimiter);
m_returned_bytes_size += output_delimiter.size();
- }
+ }
+
+
+//TODO config / default-value
+#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024)
+ if(m_fp_s3select_result_format && m_fp_s3select_header_format)
+ {
+ if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT)
+ {//there are systems that might resject the response due to its size.
+ m_fp_s3select_result_format(result);
+ m_fp_s3select_header_format(result);
+ }
+ }
}
+ void flush_sql_result(std::string& result)
+ {//purpose: flush remaining data reside in the buffer
+ if(m_fp_s3select_result_format && m_fp_s3select_header_format)
+ {
+ m_fp_s3select_result_format(result);
+ m_fp_s3select_header_format(result);
+ }
+ }
+
Status getMatchRow( std::string& result)
{
multi_values projections_resuls;
@@ -2621,8 +2690,6 @@
}; //base_s3object
-//TODO config / default-value
-#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024)
class csv_object : public base_s3object
{
@@ -2935,16 +3002,6 @@
return -1;
}
}
-
- if(m_fp_s3select_result_format && m_fp_s3select_header_format)
- {
- if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT)
- {//there are systems that might resject the response due to its size.
- m_fp_s3select_result_format(result);
- m_fp_s3select_header_format(result);
- }
- }
-
if (m_sql_processing_status == Status::END_OF_STREAM)
{
break;
@@ -2960,12 +3017,7 @@
} while (true);
- if(m_fp_s3select_result_format && m_fp_s3select_header_format)
- { //note: it may produce empty response(more the once)
- //upon empty result, it should return *only* upon last call.
- m_fp_s3select_result_format(result);
- m_fp_s3select_header_format(result);
- }
+ flush_sql_result(result);
return 0;
}
@@ -2976,7 +3028,7 @@
{
private:
- parquet_file_parser* object_reader;
+ std::unique_ptr<parquet_file_parser> object_reader;
parquet_file_parser::column_pos_t m_where_clause_columns;
parquet_file_parser::column_pos_t m_projections_columns;
std::vector<parquet_file_parser::parquet_value_t> m_predicate_values;
@@ -2985,11 +3037,14 @@
public:
- parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query),object_reader(nullptr)
+ class csv_definitions : public s3select_csv_definitions
+ {};
+
+ parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query)
{
try{
- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr
+ object_reader = std::make_unique<parquet_file_parser>(parquet_file_name,rgw);
} catch(std::exception &e)
{
throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL);
@@ -2998,7 +3053,7 @@
parquet_query_setting(nullptr);
}
- parquet_object() : base_s3object(nullptr),object_reader(nullptr)
+ parquet_object() : base_s3object(nullptr), not_to_increase_first_time(true)
{}
void parquet_query_setting(s3select *s3_query)
@@ -3020,13 +3075,7 @@
}
~parquet_object()
- {
- if(object_reader != nullptr)
- {
- delete object_reader;
- }
-
- }
+ {}
std::string get_error_description()
{
@@ -3038,11 +3087,11 @@
return m_s3_select != nullptr;
}
- void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) //TODO duplicate code
+ void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw,csv_definitions parquet) //TODO duplicate code
{
try{
-
- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr
+ m_csv_defintion = parquet;
+ object_reader = std::make_unique<parquet_file_parser>(parquet_file_name,rgw);
} catch(std::exception &e)
{
throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL);
@@ -3081,26 +3130,6 @@
}
}
-#define S3SELECT_RESPONSE_SIZE_LIMIT (4 * 1024 * 1024)
- if (result.size() > S3SELECT_RESPONSE_SIZE_LIMIT)
- {//AWS-cli limits response size the following callbacks send response upon some threshold
- if(m_fp_s3select_result_format)
- m_fp_s3select_result_format(result);
-
- if (!is_end_of_stream() && (get_sql_processing_status() != Status::LIMIT_REACHED))
- {
- if(m_fp_s3select_header_format)
- m_fp_s3select_header_format(result);
- }
- }
- else
- {
- if (is_end_of_stream() || (get_sql_processing_status() == Status::LIMIT_REACHED))
- {
- if(m_fp_s3select_result_format)
- m_fp_s3select_result_format(result);
- }
- }
//TODO is_end_of_stream() required?
if (get_sql_processing_status() == Status::END_OF_STREAM || is_end_of_stream() || get_sql_processing_status() == Status::LIMIT_REACHED)
@@ -3110,6 +3139,7 @@
} while (1);
+ flush_sql_result(result);
return 0;
}
@@ -3181,10 +3211,6 @@
f_push_key_value_into_scratch_area_per_star_operation = [this](s3selectEngine::scratch_area::json_key_value_t& key_value)
{return push_key_value_into_scratch_area_per_star_operation(key_value);};
- //setting the container for all json-variables, to be extracted by the json reader
- JsonHandler.set_statement_json_variables(query->get_json_variables_access());
-
-
//calling to getMatchRow. processing a single row per each call.
JsonHandler.set_s3select_processing_callback(f_sql);
//upon excat match between input-json-key-path and sql-statement-variable-path the callback pushes to scratch area
@@ -3226,6 +3252,9 @@
}
m_sa->set_parquet_type();//TODO json type
+
+ //setting the container for all json-variables, to be extracted by the json reader
+ JsonHandler.set_statement_json_variables(query->get_json_variables_access());
}
json_object(s3select* query):base_s3object(query),m_processed_bytes(0),m_end_of_stream(false),m_row_count(0),star_operation_ind(false),m_init_json_processor_ind(false)
@@ -3293,7 +3322,10 @@
int push_key_value_into_scratch_area_per_star_operation(s3selectEngine::scratch_area::json_key_value_t& key_value)
{
- m_sa->get_star_operation_cont()->push_back( key_value );
+ //upon star-operation on nested JSON, there could be many keys in a single row (actually, there is no limitation).
+ //for many cases these keys are duplicated in the scope of a single-row (row is defined according to SQL statement).
+ //the following routine saves only unique keys.
+ m_sa->json_push_key_value_per_star_operation(key_value);
return 0;
}
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h ceph-19.2.2/src/s3select/include/s3select_json_parser.h
--- ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select_json_parser.h 2025-07-07 10:22:00.000000000 -0400
@@ -340,12 +340,12 @@
if(reader_position_state().required_path.size())//current state is a key
{
- std::vector<std::string>* filter = &reader_position_state().required_path;
+ std::vector<std::string>* projection_key_path = &reader_position_state().required_path;
auto required_key_depth_size = reader_position_state().required_key_depth_size;
if(std::equal((*key_path).begin()+(*from_clause).size() + required_key_depth_size, //key-path-start-point + from-clause-depth-size + key-depth
(*key_path).end(),
- (*filter).begin(),
- (*filter).end(), iequal_predicate))
+ (*projection_key_path).begin(),
+ (*projection_key_path).end(), iequal_predicate))
{
increase_current_state();//key match according to user request, advancing to the next state
}
@@ -640,15 +640,45 @@
m_current_depth_non_anonymous++;
}
- if(from_clause.size() == 0 || std::equal(key_path.begin(), key_path.end(), from_clause.begin(), from_clause.end(), iequal_predicate)) {
- prefix_match = true;
- }
-
variable_match_operations.key();
return true;
}
+ template<typename predicate>
+ bool from_clause_matcher(std::vector<std::string>& _key_path,
+ std::vector<std::string>& _from_clause,
+ predicate p)
+ {
+ //iterate on both path's
+ //upon a part of from-clause is '*' it consider 'equal' to the counter part (projection), should skip to the next part
+ //
+ //from-clause = a.*.c ; projection-key = a.b.c ; since the '*' is on the secod position
+ //it means b=exists-in-from-clause, it should skip to next part (the third).
+
+ std::vector<std::string>::iterator it_key_path = _key_path.begin();
+ std::vector<std::string>::iterator it_from_clause = _from_clause.begin();
+
+ while(it_from_clause != _from_clause.end())
+ {
+ if (it_key_path == _key_path.end()) return false;
+
+ if ((it_from_clause->compare("*")==0) || p(*it_key_path,*it_from_clause))
+ {it_key_path++; it_from_clause++;}
+ else
+ return false;
+ };
+ return true;
+
+ };
+
+
+ void set_prefix_match(){
+ if(from_clause.size() == 0 || from_clause_matcher(key_path, from_clause, iequal_predicate)) {
+ prefix_match = true; //it is not prefix_match in the case its a key/value . it is a prefix match in the case it is a key of array or key of an object
+ }
+ }
+
bool is_already_row_started()
{
if(state == row_state::OBJECT_START_ROW || state == row_state::ARRAY_START_ROW)
@@ -658,6 +688,8 @@
}
bool StartObject() {
+ set_prefix_match();
+
json_element_state.push_back(OBJECT_STATE);
m_current_depth++;
if(key_path.size()){
@@ -690,6 +722,8 @@
}
bool StartArray() {
+ set_prefix_match();
+
json_element_state.push_back(ARRAY_STATE);
m_current_depth++;
if(key_path.size()){
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_oper.h ceph-19.2.2/src/s3select/include/s3select_oper.h
--- ceph-19.2.2/src/s3select.orig/include/s3select_oper.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select_oper.h 2025-07-07 10:22:00.000000000 -0400
@@ -592,9 +592,9 @@
value_En_t _type() const { return type; }
- void set_json_key_path(std::vector<std::string>& key_path)
+ void set_json_key_path(const std::vector<std::string>& key_path)
{
- m_json_key = key_path;
+ m_json_key = key_path;//TODO not efficient
}
const char* to_string() //TODO very intensive , must improve this
@@ -693,9 +693,10 @@
m_str_value = o.m_str_value;
__val.str = m_str_value.data();
}
- else if(o.__val.str)
+ else if(o.__val.str)//it is done upon using the set_string_nocopy
{
- __val.str = o.__val.str;
+ m_str_value.assign(o.__val.str);//need to create a copy for this
+ __val.str = m_str_value.data();
}
}
else
@@ -717,9 +718,10 @@
m_str_value = o.m_str_value;
__val.str = m_str_value.data();
}
- else if(o.__val.str)
+ else if(o.__val.str)//it is done upon using the set_string_nocopy
{
- __val.str = o.__val.str;
+ m_str_value.assign(o.__val.str);
+ __val.str = m_str_value.data();
}
}
else
@@ -1137,6 +1139,54 @@
}
}
+/////////////////////////
+#include <unordered_map>
+class json_key_value_store
+{
+//hash function for std::vector<std::string>
+struct VectorStringHash {
+ std::size_t operator()(const std::vector<std::string>& vec) const {
+ std::size_t seed = 0;
+ for (const auto& str : vec) {
+ for (char c : str) {
+ // boost::hash_combine_size_t
+ seed ^= std::hash<char>()(c) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
+ }
+ }
+ return seed;
+ }
+};
+
+//equality comparison function for std::vector<std::string>
+struct VectorStringEqual {
+ bool operator()(const std::vector<std::string>& lhs, const std::vector<std::string>& rhs) const {
+ return lhs == rhs;
+ }
+};
+
+public:
+
+//NOTE: using unordered_map(and also std::map) means that output result is NOT as the input order (TODO consider using an order column)
+//order means, order of keys, as appear in the input. order has no affect on the correctness of results.
+//the current flow, for identicals keys, save the first key/value and ignore all the rest.
+//
+//this type of container avoid huge memeory consumption upon a very big row(e.g. { "root" : [ long-and-nested-objects ] } )
+//a 'select * from s3object[*];' has to scan the entire "root" object before returing a result,
+//the star-operation must upload all keys-values for that row.
+//without a unique-key machanism it will consume large amount of memory(entire input JSON object).
+//
+//NOTE: upon huge JSON object containing unique-keys(random names), it will cause high memory consumption.
+//TODO:consider using container size limitation.
+typedef std::unordered_map<std::vector<std::string>, value, VectorStringHash, VectorStringEqual> _unorder_map_t;
+
+_unorder_map_t json_key_value_store;
+
+void insert(const std::vector<std::string>& key,value v)
+{
+ json_key_value_store.emplace( std::pair<std::vector<std::string>,value>(key,v) );
+}
+
+};
class scratch_area
{
@@ -1155,8 +1205,8 @@
public:
typedef std::pair<std::vector<std::string>,value> json_key_value_t;
- typedef std::vector< json_key_value_t > json_star_op_cont_t;
- json_star_op_cont_t m_json_star_operation;
+
+ json_key_value_store m_json_star_operation;
scratch_area():m_upper_bound(-1),parquet_type(false),buff_loc(0),max_json_idx(-1)
{
@@ -1168,14 +1218,20 @@
delete m_schema_values;
}
- json_star_op_cont_t* get_star_operation_cont()
+ json_key_value_store::_unorder_map_t* get_star_operation_cont()
+ {
+ return &m_json_star_operation.json_key_value_store;
+ }
+
+ void json_push_key_value_per_star_operation(json_key_value_t& key_value)
{
- return &m_json_star_operation;
+ //identical keys(and thier values) are not saved.
+ m_json_star_operation.insert(key_value.first,key_value.second);
}
void clear_data()
{
- m_json_star_operation.clear();
+ m_json_star_operation.json_key_value_store.clear();
for(int i=0;i<=max_json_idx;i++)
{
(*m_schema_values)[i].setnull();
@@ -1857,8 +1913,8 @@
else
{
m_scratch->get_column_value(column_pos,var_value);
- //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL
- if (var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){
+ //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL(CSV data source)
+ if (!is_json_statement() && var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){
var_value.setnull();//TODO is it correct for Parquet
}
}
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h
--- ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h 2025-07-07 13:08:00.747275733 -0400
@@ -434,7 +434,7 @@
#if ARROW_VERSION_MAJOR < 9
fd_ = -1;
#else
- fd_.Close();
+ return fd_.Close();
#endif
//RETURN_NOT_OK(::arrow::internal::FileClose(fd));
}
@@ -589,17 +589,12 @@
public:
~ReadableFileImpl()
- {
- if(IMPL != nullptr)
- {
- delete IMPL;
- }
- }
+ {}
#ifdef CEPH_USE_FS
- explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=new OSFile();}
+ explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=std::make_unique<OSFile>();}
#endif
- explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=new RGWimpl(rgw);}
+ explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=std::make_unique<RGWimpl>(rgw);}
Status Open(const std::string& path) { return IMPL->OpenReadable(path); }
@@ -649,7 +644,7 @@
return Status::OK();
}
- ObjectInterface *IMPL;//TODO to declare in ObjectInterface
+ std::unique_ptr<ObjectInterface> IMPL;
private:
@@ -752,6 +747,30 @@
std::unique_ptr<Contents> contents_;
};
+#define RGW_default_buffer_size 1024*1024*1024
+class S3select_Config {
+public:
+ static S3select_Config& getInstance() {
+ static S3select_Config instance;
+ return instance;
+ }
+
+ void set_s3select_reader_properties(uint64_t value) { this->m_reader_properties = value; }
+ uint64_t get_s3select_reader_properties() const { return m_reader_properties; }
+
+private:
+ uint64_t m_reader_properties;
+ S3select_Config() : m_reader_properties(RGW_default_buffer_size) {} // Private constructor
+};
+
+ReaderProperties s3select_reader_properties() {
+
+ static ReaderProperties default_reader_properties;
+ default_reader_properties.enable_buffered_stream();
+ default_reader_properties.set_buffer_size(S3select_Config::getInstance().get_s3select_reader_properties());
+ return default_reader_properties;
+}
+
class PARQUET_EXPORT ParquetFileReader {
public:
// Declare a virtual class 'Contents' to aid dependency injection and more
@@ -760,7 +779,7 @@
struct PARQUET_EXPORT Contents {
static std::unique_ptr<Contents> Open(
std::shared_ptr<::arrow::io::RandomAccessFile> source,
- const ReaderProperties& props = default_reader_properties(),
+ const ReaderProperties& props = s3select_reader_properties(),
std::shared_ptr<FileMetaData> metadata = NULLPTR);
virtual ~Contents() = default;
@@ -781,21 +800,21 @@
ARROW_DEPRECATED("Use arrow::io::RandomAccessFile version")
static std::unique_ptr<ParquetFileReader> Open(
std::unique_ptr<RandomAccessSource> source,
- const ReaderProperties& props = default_reader_properties(),
+ const ReaderProperties& props = s3select_reader_properties(),
std::shared_ptr<FileMetaData> metadata = NULLPTR);
// Create a file reader instance from an Arrow file object. Thread-safety is
// the responsibility of the file implementation
static std::unique_ptr<ParquetFileReader> Open(
std::shared_ptr<::arrow::io::RandomAccessFile> source,
- const ReaderProperties& props = default_reader_properties(),
+ const ReaderProperties& props = s3select_reader_properties(),
std::shared_ptr<FileMetaData> metadata = NULLPTR);
// API Convenience to open a serialized Parquet file on disk, using Arrow IO
// interfaces.
static std::unique_ptr<ParquetFileReader> OpenFile(
const std::string& path,s3selectEngine::rgw_s3select_api* rgw, bool memory_map = true,
- const ReaderProperties& props = default_reader_properties(),
+ const ReaderProperties& props = s3select_reader_properties(),
std::shared_ptr<FileMetaData> metadata = NULLPTR
);
@@ -983,6 +1002,7 @@
throw ParquetException("Encrypted files cannot contain more than 32767 row groups");
}
@@ -1611,15 +796,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
}
private:
@@ -1039,7 +1078,7 @@
class SerializedFile : public ParquetFileReader::Contents {
public:
SerializedFile(std::shared_ptr<ArrowInputFile> source,
- const ReaderProperties& props = default_reader_properties())
+ const ReaderProperties& props = s3select_reader_properties())
: source_(std::move(source)), properties_(props) {
PARQUET_ASSIGN_OR_THROW(source_size_, source_->GetSize());
}
@@ -1052,7 +1091,9 @@
}
@@ -1648,21 +824,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len;
uint32_t metadata_len = footer_len - crypto_metadata_len;
@@ -1244,9 +1293,13 @@
std::to_string(metadata_buffer->size()) + " bytes)");
}
+#if ARROW_VERSION_MAJOR > 9
+ file_metadata_ =
+ FileMetaData::Make(metadata_buffer->data(), &metadata_len, s3select_reader_properties(), file_decryptor_);
+#else
file_metadata_ =
FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_);
- //FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9
+#endif
}
void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter(
@@ -1259,9 +1312,18 @@
EncryptionAlgorithm algo = file_metadata_->encryption_algorithm();
// Handle AAD prefix
@@ -1682,57 +843,3 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
// set the InternalFileDecryptor in the metadata as well, as it's used
// for signature verification and for ColumnChunkMetaData creation.
#if GAL_set_file_decryptor_declare_private
@@ -1584,7 +1646,7 @@
int m_num_row_groups;
std::shared_ptr<parquet::FileMetaData> m_file_metadata;
std::unique_ptr<parquet::ceph::ParquetFileReader> m_parquet_reader;
- std::vector<column_reader_wrap*> m_column_readers;
+ std::vector<std::shared_ptr<column_reader_wrap>> m_column_readers;
s3selectEngine::rgw_s3select_api* m_rgw_s3select_api;
public:
@@ -1603,12 +1665,7 @@
}
~parquet_file_parser()
- {
- for(auto r : m_column_readers)
- {
- delete r;
- }
- }
+ {}
int load_meta_data()
{
@@ -1658,7 +1715,7 @@
}
}
- m_column_readers.push_back(new column_reader_wrap(m_parquet_reader,i));
+ m_column_readers.push_back(std::make_shared<column_reader_wrap>(m_parquet_reader,i));
}
return 0;
--- ceph-19.2.2/src/rgw/rgw_s3select.cc.orig 2025-07-07 14:24:23.371796969 -0400
+++ ceph-19.2.2/src/rgw/rgw_s3select.cc 2025-07-07 17:45:30.655554945 -0400
@@ -472,6 +472,7 @@
if (!m_s3_parquet_object.is_set()) {
//parsing the SQL statement.
s3select_syntax.parse_query(m_sql_query.c_str());
+ parquet_object::csv_definitions parquet;
m_s3_parquet_object.set_external_system_functions(fp_s3select_continue,
fp_s3select_result_format,
@@ -479,8 +480,9 @@
fp_debug_mesg);
try {
- //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object
- m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api);
+ //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object
+ // m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api);
+ m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api, parquet);
} catch(base_s3select_exception& e) {
ldpp_dout(this, 10) << "S3select: failed upon parquet-reader construction: " << e.what() << dendl;
fp_result_header_format(m_aws_response_handler.get_sql_result());
+12 -2
View File
@@ -217,8 +217,8 @@
# main package definition
#################################################################################
Name: ceph
Version: 19.2.2
Release: 2%{?dist}
Version: 19.2.3
Release: 1%{?dist}
%if 0%{?fedora} || 0%{?rhel}
Epoch: 2
%endif
@@ -328,6 +328,7 @@ BuildRequires: gperftools-devel >= 2.4
BuildRequires: libaio-devel
BuildRequires: libblkid-devel >= 2.17
BuildRequires: cryptsetup-devel
BuildRequires: libnbd-devel
BuildRequires: libcurl-devel
BuildRequires: libcap-devel
BuildRequires: libcap-ng-devel
@@ -730,6 +731,7 @@ Requires: python%{python3_pkgversion}-setuptools
%if 0%{?fedora} || 0%{?rhel} >= 9
Requires: python%{python3_pkgversion}-grpcio
Requires: python%{python3_pkgversion}-grpcio-tools
Requires: python%{python3_pkgversion}-xmltodict
%endif
%if 0%{?fedora} || 0%{?rhel}
Requires: python%{python3_pkgversion}-cherrypy
@@ -1784,6 +1786,9 @@ rm -f %{buildroot}%{_mandir}/man8/ceph-bluestore-tool.8*
rm -f %{buildroot}%{_unitdir}/ceph-osd@.service
rm -f %{buildroot}%{_unitdir}/ceph-osd.target
# Remove ceph-exporter
rm -f %{buildroot}%{_unitdir}/ceph-exporter.service
# Remove rbd-mirror
rm -f %{buildroot}%{_bindir}/rbd-mirror
rm -f %{buildroot}%{_mandir}/man8/rbd-mirror.8*
@@ -2007,6 +2012,7 @@ exit 0
%{_mandir}/man8/rbd-replay-prep.8*
%if 0%{with radosgw}
%{_mandir}/man8/rgw-orphan-list.8*
%{_mandir}/man8/rgw-gap-list.8*
%{_mandir}/man8/rgw-restore-bucket-index.8*
%endif
%dir %{_datadir}/ceph/
@@ -2343,6 +2349,7 @@ fi
%files -n ceph-exporter
%{_bindir}/ceph-exporter
%{_unitdir}/ceph-exporter.service
%files -n rbd-fuse
%{_bindir}/rbd-fuse
@@ -2939,6 +2946,9 @@ exit 0
%endif
%changelog
* Mon Jul 28 2025 Kaleb S. KEITHLEY <kkeithle[at]redhat.com> - 2:19.2.3-1
- ceph-19.2.3 GA
* Tue Jul 15 2025 Kaleb S. KEITHLEY <kkeithle[at]redhat.com> - 2:19.2.2-2
- ceph-19.2.2, rebuild with libarrow-20 and liborc-2.1.3
+1 -1
View File
@@ -1 +1 @@
SHA512 (ceph-19.2.2.tar.gz) = ee47c1cb7cb5084b87bcc5a35b3df88fb49683524bba8f2e1ced9d2f8891af53e4b5fb5aa153ed6bce31683625d9bf5176bab9f55bc71671f0e34667948f7285
SHA512 (ceph-19.2.3.tar.gz) = 278101d2df7bed5363b20c2b065d7a7b26252c8164511257e213ffaa58d509015558183de10bc9281bcbe4d9f85244bcac5bba4db9823e28df6a96d0b687d00a