|
|
|
@@ -759,824 +759,9 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h ceph-19
|
|
|
|
|
+#else
|
|
|
|
|
+#include "internal_file_decryptor_20.h"
|
|
|
|
|
+#endif
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_functions.h ceph-19.2.2/src/s3select/include/s3select_functions.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select_functions.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select_functions.h 2025-07-07 10:22:00.000000000 -0400
|
|
|
|
|
@@ -505,7 +505,7 @@
|
|
|
|
|
|
|
|
|
|
std::string print(int ident) override
|
|
|
|
|
{
|
|
|
|
|
- return std::string(0);
|
|
|
|
|
+ return std::string{};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void push_argument(base_statement* arg)
|
|
|
|
|
@@ -2268,6 +2268,17 @@
|
|
|
|
|
-- avoid timeout upon long processing #152
|
|
|
|
|
-- replace assert with an exception to avoid crashing the process #151
|
|
|
|
|
-- fix for the use-case of not-operator on a string #153
|
|
|
|
|
+
|
|
|
|
|
+-- the negation-operation may cause a wrong-build of the AST, that later may cause a crash. that operation is missing handling of several operators. #160
|
|
|
|
|
+-- the crash happened upon calling more than once the parse_query, the second call accessed an incomplete object in the AST. #160
|
|
|
|
|
+
|
|
|
|
|
+-- fix for identifies key-value as key-object or key-array #161
|
|
|
|
|
+-- wrong initialization of the JSON parser engine had caused missing a projection-key-path upon using different combinations of the from-clause path. #161
|
|
|
|
|
+-- a fix for copy-constructor, upon a JSON value is an empty string, it causes a wrong result. #161
|
|
|
|
|
+-- modification for JSON star operation, the new-type container saves only unique keys, to avoid high memory consumption. #161
|
|
|
|
|
+-- the from-clause can handle a wild-card. i.e. upon wild-card(*) it skips the corresponding part in projection-key-path. #161
|
|
|
|
|
+-- key-path may include meta-char(like a dot) select _1."i.e."[0] the "i.e." is part of the key-path. #161
|
|
|
|
|
+
|
|
|
|
|
)";
|
|
|
|
|
|
|
|
|
|
_fn_engine_version()
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select.h ceph-19.2.2/src/s3select/include/s3select.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select.h 2025-07-07 10:22:00.000000000 -0400
|
|
|
|
|
@@ -45,6 +45,43 @@
|
|
|
|
|
|
|
|
|
|
static s3select_reserved_word g_s3select_reserve_word;//read-only
|
|
|
|
|
|
|
|
|
|
+struct expr_queue {
|
|
|
|
|
+ //purpose: a warpper for std::vector for monitoring the expression queue actions.
|
|
|
|
|
+ //upon an SQL expression is resolved by the engine it is pushed into the queue for later operations
|
|
|
|
|
+ std::vector<base_statement*> m_expr_queue;
|
|
|
|
|
+
|
|
|
|
|
+ base_statement* back()
|
|
|
|
|
+ {
|
|
|
|
|
+ //expression queue must contain an element upon accessing it. upon an empty expression queue to throw an exception.
|
|
|
|
|
+ if(m_expr_queue.empty())
|
|
|
|
|
+ {
|
|
|
|
|
+ throw base_s3select_exception("expression queue is empty, abort parsing.", base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
+ }
|
|
|
|
|
+ return m_expr_queue.back();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ void push_back(base_statement* bs)
|
|
|
|
|
+ {
|
|
|
|
|
+ m_expr_queue.push_back(bs);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ void pop_back()
|
|
|
|
|
+ {
|
|
|
|
|
+ return m_expr_queue.pop_back();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ bool empty()
|
|
|
|
|
+ {
|
|
|
|
|
+ return m_expr_queue.empty();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ void clear()
|
|
|
|
|
+ {
|
|
|
|
|
+ return m_expr_queue.clear();
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
struct actionQ
|
|
|
|
|
{
|
|
|
|
|
// upon parser is accepting a token (lets say some number),
|
|
|
|
|
@@ -55,7 +92,7 @@
|
|
|
|
|
std::vector<addsub_operation::addsub_op_t> addsubQ;
|
|
|
|
|
std::vector<arithmetic_operand::cmp_t> arithmetic_compareQ;
|
|
|
|
|
std::vector<logical_operand::oplog_t> logical_compareQ;
|
|
|
|
|
- std::vector<base_statement*> exprQ;
|
|
|
|
|
+ expr_queue exprQ;
|
|
|
|
|
std::vector<base_statement*> funcQ;
|
|
|
|
|
std::vector<base_statement*> whenThenQ;
|
|
|
|
|
std::vector<base_statement*> inPredicateQ;
|
|
|
|
|
@@ -144,6 +181,12 @@
|
|
|
|
|
};
|
|
|
|
|
static push_json_from_clause g_push_json_from_clause;
|
|
|
|
|
|
|
|
|
|
+struct push_json_from_clause_key_path : public base_ast_builder
|
|
|
|
|
+{
|
|
|
|
|
+ void builder(s3select* self, const char* a, const char* b) const;
|
|
|
|
|
+};
|
|
|
|
|
+static push_json_from_clause_key_path g_push_json_from_clause_key_path;
|
|
|
|
|
+
|
|
|
|
|
struct push_limit_clause : public base_ast_builder
|
|
|
|
|
{
|
|
|
|
|
void builder(s3select* self, const char* a, const char* b) const;
|
|
|
|
|
@@ -513,6 +556,12 @@
|
|
|
|
|
|
|
|
|
|
int semantic()
|
|
|
|
|
{
|
|
|
|
|
+ if(get_projections_list().empty())
|
|
|
|
|
+ {
|
|
|
|
|
+ error_description = "projections list is empty, AST is wrong,";
|
|
|
|
|
+ throw base_s3select_exception(error_description, base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
for (const auto &e : get_projections_list())
|
|
|
|
|
{
|
|
|
|
|
e->resolve_node();
|
|
|
|
|
@@ -746,7 +795,7 @@
|
|
|
|
|
|
|
|
|
|
json_s3_object = ((S3SELECT_KW(JSON_ROOT_OBJECT)) >> *(bsc::str_p(".") >> json_path_element))[BOOST_BIND_ACTION(push_json_from_clause)];
|
|
|
|
|
|
|
|
|
|
- json_path_element = bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_")) ];
|
|
|
|
|
+ json_path_element = (bsc::lexeme_d[+( bsc::alnum_p | bsc::str_p("_") ) ] | bsc::str_p("*") | (string))[BOOST_BIND_ACTION(push_json_from_clause_key_path)];
|
|
|
|
|
|
|
|
|
|
object_path = "/" >> *( fs_type >> "/") >> fs_type;
|
|
|
|
|
|
|
|
|
|
@@ -872,11 +921,12 @@
|
|
|
|
|
|
|
|
|
|
variable = (variable_name >> "." >> variable_name) | variable_name;
|
|
|
|
|
|
|
|
|
|
+ // json_variable_name is the JSON projection, i.e. _1.a.b[10]
|
|
|
|
|
json_variable_name = bsc::str_p("_1") >> +("." >> (json_array | json_object) );
|
|
|
|
|
|
|
|
|
|
- json_object = (variable_name)[BOOST_BIND_ACTION(push_json_object)];
|
|
|
|
|
+ json_object = (variable_name | string)[BOOST_BIND_ACTION(push_json_object)];
|
|
|
|
|
|
|
|
|
|
- json_array = (variable_name >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)];
|
|
|
|
|
+ json_array = ((variable_name | string) >> +(bsc::str_p("[") >> number[BOOST_BIND_ACTION(push_array_number)] >> bsc::str_p("]")) )[BOOST_BIND_ACTION(push_json_array_name)];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@@ -936,35 +986,33 @@
|
|
|
|
|
self->getAction()->exprQ.clear();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
-void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
+std::string json_path_remove_double_quote(const char* a, const char* b)
|
|
|
|
|
+{
|
|
|
|
|
+ //upon query accessing key which contains meta-char, it must use string-construct(double quotes),
|
|
|
|
|
+ //the engine should remove double quotes for later processing.
|
|
|
|
|
+ std::string token(a, b);
|
|
|
|
|
+ if(*a == '"') //TODO single quote ?
|
|
|
|
|
+ {
|
|
|
|
|
+ std::string tmp = token.substr(1,token.find('"',1)-1);
|
|
|
|
|
+ token = tmp;
|
|
|
|
|
+ }
|
|
|
|
|
+ return token;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+void push_json_from_clause_key_path::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
{
|
|
|
|
|
- std::string token(a, b),table_name,alias_name;
|
|
|
|
|
|
|
|
|
|
- //TODO handle the star-operation ('*') in from-clause. build the parameters for json-reader search-api's.
|
|
|
|
|
- std::vector<std::string> variable_key_path;
|
|
|
|
|
- const char* delimiter = ".";
|
|
|
|
|
- auto pos = token.find(delimiter);
|
|
|
|
|
+ std::string token = json_path_remove_double_quote(a,b);
|
|
|
|
|
+ self->getAction()->json_from_clause.push_back(token);
|
|
|
|
|
+
|
|
|
|
|
+}
|
|
|
|
|
|
|
|
|
|
- if(pos != std::string::npos)
|
|
|
|
|
- {
|
|
|
|
|
- token = token.substr(strlen(JSON_ROOT_OBJECT)+1,token.size());
|
|
|
|
|
- pos = token.find(delimiter);
|
|
|
|
|
- do
|
|
|
|
|
- {
|
|
|
|
|
- variable_key_path.push_back(token.substr(0,pos));
|
|
|
|
|
- if(pos != std::string::npos)
|
|
|
|
|
- token = token.substr(pos+1,token.size());
|
|
|
|
|
- else
|
|
|
|
|
- token = "";
|
|
|
|
|
- pos = token.find(delimiter);
|
|
|
|
|
- }while(token.size());
|
|
|
|
|
- }
|
|
|
|
|
- else
|
|
|
|
|
+void push_json_from_clause::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
+{
|
|
|
|
|
+ if(self->getAction()->json_from_clause.size() == 0)
|
|
|
|
|
{
|
|
|
|
|
- variable_key_path.push_back(JSON_ROOT_OBJECT);
|
|
|
|
|
+ self->getAction()->json_from_clause.push_back(JSON_ROOT_OBJECT);
|
|
|
|
|
}
|
|
|
|
|
-
|
|
|
|
|
- self->getAction()->json_from_clause = variable_key_path;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void push_limit_clause::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
@@ -1086,7 +1134,6 @@
|
|
|
|
|
{//purpose: handle the use case of json-variable structure (_1.a.b.c)
|
|
|
|
|
|
|
|
|
|
std::string token(a, b);
|
|
|
|
|
- std::vector<std::string> variable_key_path;
|
|
|
|
|
|
|
|
|
|
//the following flow determine the index per json variable reside on statement.
|
|
|
|
|
//per each discovered json_variable, it search the json-variables-vector whether it already exists.
|
|
|
|
|
@@ -1116,7 +1163,8 @@
|
|
|
|
|
|
|
|
|
|
void push_json_array_name::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
{
|
|
|
|
|
- std::string token(a, b);
|
|
|
|
|
+ std::string token = json_path_remove_double_quote(a,b);
|
|
|
|
|
+
|
|
|
|
|
size_t found = token.find("[");
|
|
|
|
|
std::string array_name = token.substr(0,found);
|
|
|
|
|
|
|
|
|
|
@@ -1143,7 +1191,7 @@
|
|
|
|
|
|
|
|
|
|
void push_json_object::builder(s3select* self, const char* a, const char* b) const
|
|
|
|
|
{
|
|
|
|
|
- std::string token(a, b);
|
|
|
|
|
+ std::string token = json_path_remove_double_quote(a,b);
|
|
|
|
|
|
|
|
|
|
//DEBUG - TEMP std::cout << "push_json_object " << token << std::endl;
|
|
|
|
|
|
|
|
|
|
@@ -1385,7 +1433,7 @@
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, expression is missing."), base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
//upon NOT operator, the logical and arithmetical operators are "tagged" to negate result.
|
|
|
|
|
@@ -1394,7 +1442,8 @@
|
|
|
|
|
logical_operand* f = S3SELECT_NEW(self, logical_operand, pred);
|
|
|
|
|
self->getAction()->exprQ.push_back(f);
|
|
|
|
|
}
|
|
|
|
|
- else if (dynamic_cast<__function*>(pred) || dynamic_cast<negate_function_operation*>(pred) || dynamic_cast<variable*>(pred))
|
|
|
|
|
+ else if (dynamic_cast<__function*>(pred) || dynamic_cast<negate_function_operation*>(pred) || dynamic_cast<variable*>(pred)
|
|
|
|
|
+ || dynamic_cast<addsub_operation*>(pred) || dynamic_cast<mulldiv_operation*>(pred))
|
|
|
|
|
{
|
|
|
|
|
negate_function_operation* nf = S3SELECT_NEW(self, negate_function_operation, pred);
|
|
|
|
|
self->getAction()->exprQ.push_back(nf);
|
|
|
|
|
@@ -1406,7 +1455,7 @@
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
- throw base_s3select_exception(std::string("failed to create AST for NOT operator"), base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
+ throw base_s3select_exception(std::string("failed to create AST for NOT operator, appropiate operator is missing"), base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -2409,6 +2458,7 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
result.append(res->to_string());
|
|
|
|
|
+ //TODO the strlen should replace with the size of the string(performance perspective)
|
|
|
|
|
m_returned_bytes_size += strlen(res->to_string());
|
|
|
|
|
++j;
|
|
|
|
|
}
|
|
|
|
|
@@ -2425,6 +2475,8 @@
|
|
|
|
|
if(m_csv_defintion.output_json_format && projections_resuls.values.size()) {
|
|
|
|
|
json_result_format(projections_resuls, result, output_delimiter);
|
|
|
|
|
result.append(output_row_delimiter);
|
|
|
|
|
+ //TODO to add asneeded
|
|
|
|
|
+ //TODO "flush" the result string
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -2444,10 +2496,6 @@
|
|
|
|
|
set_processing_time_error();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- if(m_fp_ext_debug_mesg)
|
|
|
|
|
- m_fp_ext_debug_mesg(column_result.data());
|
|
|
|
|
-
|
|
|
|
|
if (m_csv_defintion.quote_fields_always) {
|
|
|
|
|
std::ostringstream quoted_result;
|
|
|
|
|
quoted_result << std::quoted(column_result,m_csv_defintion.output_quot_char, m_csv_defintion.escape_char);
|
|
|
|
|
@@ -2475,9 +2523,30 @@
|
|
|
|
|
if(!m_aggr_flow) {
|
|
|
|
|
result.append(output_row_delimiter);
|
|
|
|
|
m_returned_bytes_size += output_delimiter.size();
|
|
|
|
|
- }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+//TODO config / default-value
|
|
|
|
|
+#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024)
|
|
|
|
|
+ if(m_fp_s3select_result_format && m_fp_s3select_header_format)
|
|
|
|
|
+ {
|
|
|
|
|
+ if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT)
|
|
|
|
|
+ {//there are systems that might resject the response due to its size.
|
|
|
|
|
+ m_fp_s3select_result_format(result);
|
|
|
|
|
+ m_fp_s3select_header_format(result);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
+ void flush_sql_result(std::string& result)
|
|
|
|
|
+ {//purpose: flush remaining data reside in the buffer
|
|
|
|
|
+ if(m_fp_s3select_result_format && m_fp_s3select_header_format)
|
|
|
|
|
+ {
|
|
|
|
|
+ m_fp_s3select_result_format(result);
|
|
|
|
|
+ m_fp_s3select_header_format(result);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
Status getMatchRow( std::string& result)
|
|
|
|
|
{
|
|
|
|
|
multi_values projections_resuls;
|
|
|
|
|
@@ -2621,8 +2690,6 @@
|
|
|
|
|
|
|
|
|
|
}; //base_s3object
|
|
|
|
|
|
|
|
|
|
-//TODO config / default-value
|
|
|
|
|
-#define CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT (64 * 1024)
|
|
|
|
|
class csv_object : public base_s3object
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
@@ -2935,16 +3002,6 @@
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
-
|
|
|
|
|
- if(m_fp_s3select_result_format && m_fp_s3select_header_format)
|
|
|
|
|
- {
|
|
|
|
|
- if (result.size() > CSV_INPUT_TYPE_RESPONSE_SIZE_LIMIT)
|
|
|
|
|
- {//there are systems that might resject the response due to its size.
|
|
|
|
|
- m_fp_s3select_result_format(result);
|
|
|
|
|
- m_fp_s3select_header_format(result);
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
if (m_sql_processing_status == Status::END_OF_STREAM)
|
|
|
|
|
{
|
|
|
|
|
break;
|
|
|
|
|
@@ -2960,12 +3017,7 @@
|
|
|
|
|
|
|
|
|
|
} while (true);
|
|
|
|
|
|
|
|
|
|
- if(m_fp_s3select_result_format && m_fp_s3select_header_format)
|
|
|
|
|
- { //note: it may produce empty response(more the once)
|
|
|
|
|
- //upon empty result, it should return *only* upon last call.
|
|
|
|
|
- m_fp_s3select_result_format(result);
|
|
|
|
|
- m_fp_s3select_header_format(result);
|
|
|
|
|
- }
|
|
|
|
|
+ flush_sql_result(result);
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
@@ -2976,7 +3028,7 @@
|
|
|
|
|
{
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
- parquet_file_parser* object_reader;
|
|
|
|
|
+ std::unique_ptr<parquet_file_parser> object_reader;
|
|
|
|
|
parquet_file_parser::column_pos_t m_where_clause_columns;
|
|
|
|
|
parquet_file_parser::column_pos_t m_projections_columns;
|
|
|
|
|
std::vector<parquet_file_parser::parquet_value_t> m_predicate_values;
|
|
|
|
|
@@ -2985,11 +3037,14 @@
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
- parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query),object_reader(nullptr)
|
|
|
|
|
+ class csv_definitions : public s3select_csv_definitions
|
|
|
|
|
+ {};
|
|
|
|
|
+
|
|
|
|
|
+ parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) : base_s3object(s3_query)
|
|
|
|
|
{
|
|
|
|
|
try{
|
|
|
|
|
|
|
|
|
|
- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr
|
|
|
|
|
+ object_reader = std::make_unique<parquet_file_parser>(parquet_file_name,rgw);
|
|
|
|
|
} catch(std::exception &e)
|
|
|
|
|
{
|
|
|
|
|
throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
@@ -2998,7 +3053,7 @@
|
|
|
|
|
parquet_query_setting(nullptr);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- parquet_object() : base_s3object(nullptr),object_reader(nullptr)
|
|
|
|
|
+ parquet_object() : base_s3object(nullptr), not_to_increase_first_time(true)
|
|
|
|
|
{}
|
|
|
|
|
|
|
|
|
|
void parquet_query_setting(s3select *s3_query)
|
|
|
|
|
@@ -3020,13 +3075,7 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
~parquet_object()
|
|
|
|
|
- {
|
|
|
|
|
- if(object_reader != nullptr)
|
|
|
|
|
- {
|
|
|
|
|
- delete object_reader;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
- }
|
|
|
|
|
+ {}
|
|
|
|
|
|
|
|
|
|
std::string get_error_description()
|
|
|
|
|
{
|
|
|
|
|
@@ -3038,11 +3087,11 @@
|
|
|
|
|
return m_s3_select != nullptr;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw) //TODO duplicate code
|
|
|
|
|
+ void set_parquet_object(std::string parquet_file_name, s3select *s3_query,s3selectEngine::rgw_s3select_api* rgw,csv_definitions parquet) //TODO duplicate code
|
|
|
|
|
{
|
|
|
|
|
try{
|
|
|
|
|
-
|
|
|
|
|
- object_reader = new parquet_file_parser(parquet_file_name,rgw); //TODO uniq ptr
|
|
|
|
|
+ m_csv_defintion = parquet;
|
|
|
|
|
+ object_reader = std::make_unique<parquet_file_parser>(parquet_file_name,rgw);
|
|
|
|
|
} catch(std::exception &e)
|
|
|
|
|
{
|
|
|
|
|
throw base_s3select_exception(std::string("failure while processing parquet meta-data ") + std::string(e.what()) ,base_s3select_exception::s3select_exp_en_t::FATAL);
|
|
|
|
|
@@ -3081,26 +3130,6 @@
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
-#define S3SELECT_RESPONSE_SIZE_LIMIT (4 * 1024 * 1024)
|
|
|
|
|
- if (result.size() > S3SELECT_RESPONSE_SIZE_LIMIT)
|
|
|
|
|
- {//AWS-cli limits response size the following callbacks send response upon some threshold
|
|
|
|
|
- if(m_fp_s3select_result_format)
|
|
|
|
|
- m_fp_s3select_result_format(result);
|
|
|
|
|
-
|
|
|
|
|
- if (!is_end_of_stream() && (get_sql_processing_status() != Status::LIMIT_REACHED))
|
|
|
|
|
- {
|
|
|
|
|
- if(m_fp_s3select_header_format)
|
|
|
|
|
- m_fp_s3select_header_format(result);
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
- else
|
|
|
|
|
- {
|
|
|
|
|
- if (is_end_of_stream() || (get_sql_processing_status() == Status::LIMIT_REACHED))
|
|
|
|
|
- {
|
|
|
|
|
- if(m_fp_s3select_result_format)
|
|
|
|
|
- m_fp_s3select_result_format(result);
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
|
|
|
|
|
//TODO is_end_of_stream() required?
|
|
|
|
|
if (get_sql_processing_status() == Status::END_OF_STREAM || is_end_of_stream() || get_sql_processing_status() == Status::LIMIT_REACHED)
|
|
|
|
|
@@ -3110,6 +3139,7 @@
|
|
|
|
|
|
|
|
|
|
} while (1);
|
|
|
|
|
|
|
|
|
|
+ flush_sql_result(result);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@@ -3181,10 +3211,6 @@
|
|
|
|
|
f_push_key_value_into_scratch_area_per_star_operation = [this](s3selectEngine::scratch_area::json_key_value_t& key_value)
|
|
|
|
|
{return push_key_value_into_scratch_area_per_star_operation(key_value);};
|
|
|
|
|
|
|
|
|
|
- //setting the container for all json-variables, to be extracted by the json reader
|
|
|
|
|
- JsonHandler.set_statement_json_variables(query->get_json_variables_access());
|
|
|
|
|
-
|
|
|
|
|
-
|
|
|
|
|
//calling to getMatchRow. processing a single row per each call.
|
|
|
|
|
JsonHandler.set_s3select_processing_callback(f_sql);
|
|
|
|
|
//upon excat match between input-json-key-path and sql-statement-variable-path the callback pushes to scratch area
|
|
|
|
|
@@ -3226,6 +3252,9 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
m_sa->set_parquet_type();//TODO json type
|
|
|
|
|
+
|
|
|
|
|
+ //setting the container for all json-variables, to be extracted by the json reader
|
|
|
|
|
+ JsonHandler.set_statement_json_variables(query->get_json_variables_access());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
json_object(s3select* query):base_s3object(query),m_processed_bytes(0),m_end_of_stream(false),m_row_count(0),star_operation_ind(false),m_init_json_processor_ind(false)
|
|
|
|
|
@@ -3293,7 +3322,10 @@
|
|
|
|
|
|
|
|
|
|
int push_key_value_into_scratch_area_per_star_operation(s3selectEngine::scratch_area::json_key_value_t& key_value)
|
|
|
|
|
{
|
|
|
|
|
- m_sa->get_star_operation_cont()->push_back( key_value );
|
|
|
|
|
+ //upon star-operation on nested JSON, there could be many keys in a single row (actually, there is no limitation).
|
|
|
|
|
+ //for many cases these keys are duplicated in the scope of a single-row (row is defined according to SQL statement).
|
|
|
|
|
+ //the following routine saves only unique keys.
|
|
|
|
|
+ m_sa->json_push_key_value_per_star_operation(key_value);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h ceph-19.2.2/src/s3select/include/s3select_json_parser.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select_json_parser.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select_json_parser.h 2025-07-07 10:22:00.000000000 -0400
|
|
|
|
|
@@ -340,12 +340,12 @@
|
|
|
|
|
|
|
|
|
|
if(reader_position_state().required_path.size())//current state is a key
|
|
|
|
|
{
|
|
|
|
|
- std::vector<std::string>* filter = &reader_position_state().required_path;
|
|
|
|
|
+ std::vector<std::string>* projection_key_path = &reader_position_state().required_path;
|
|
|
|
|
auto required_key_depth_size = reader_position_state().required_key_depth_size;
|
|
|
|
|
if(std::equal((*key_path).begin()+(*from_clause).size() + required_key_depth_size, //key-path-start-point + from-clause-depth-size + key-depth
|
|
|
|
|
(*key_path).end(),
|
|
|
|
|
- (*filter).begin(),
|
|
|
|
|
- (*filter).end(), iequal_predicate))
|
|
|
|
|
+ (*projection_key_path).begin(),
|
|
|
|
|
+ (*projection_key_path).end(), iequal_predicate))
|
|
|
|
|
{
|
|
|
|
|
increase_current_state();//key match according to user request, advancing to the next state
|
|
|
|
|
}
|
|
|
|
|
@@ -640,15 +640,45 @@
|
|
|
|
|
m_current_depth_non_anonymous++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- if(from_clause.size() == 0 || std::equal(key_path.begin(), key_path.end(), from_clause.begin(), from_clause.end(), iequal_predicate)) {
|
|
|
|
|
- prefix_match = true;
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
variable_match_operations.key();
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
+ template<typename predicate>
|
|
|
|
|
+ bool from_clause_matcher(std::vector<std::string>& _key_path,
|
|
|
|
|
+ std::vector<std::string>& _from_clause,
|
|
|
|
|
+ predicate p)
|
|
|
|
|
+ {
|
|
|
|
|
+ //iterate on both path's
|
|
|
|
|
+ //upon a part of from-clause is '*' it consider 'equal' to the counter part (projection), should skip to the next part
|
|
|
|
|
+ //
|
|
|
|
|
+ //from-clause = a.*.c ; projection-key = a.b.c ; since the '*' is on the secod position
|
|
|
|
|
+ //it means b=exists-in-from-clause, it should skip to next part (the third).
|
|
|
|
|
+
|
|
|
|
|
+ std::vector<std::string>::iterator it_key_path = _key_path.begin();
|
|
|
|
|
+ std::vector<std::string>::iterator it_from_clause = _from_clause.begin();
|
|
|
|
|
+
|
|
|
|
|
+ while(it_from_clause != _from_clause.end())
|
|
|
|
|
+ {
|
|
|
|
|
+ if (it_key_path == _key_path.end()) return false;
|
|
|
|
|
+
|
|
|
|
|
+ if ((it_from_clause->compare("*")==0) || p(*it_key_path,*it_from_clause))
|
|
|
|
|
+ {it_key_path++; it_from_clause++;}
|
|
|
|
|
+ else
|
|
|
|
|
+ return false;
|
|
|
|
|
+ };
|
|
|
|
|
+ return true;
|
|
|
|
|
+
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
|
|
+ void set_prefix_match(){
|
|
|
|
|
+ if(from_clause.size() == 0 || from_clause_matcher(key_path, from_clause, iequal_predicate)) {
|
|
|
|
|
+ prefix_match = true; //it is not prefix_match in the case its a key/value . it is a prefix match in the case it is a key of array or key of an object
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
bool is_already_row_started()
|
|
|
|
|
{
|
|
|
|
|
if(state == row_state::OBJECT_START_ROW || state == row_state::ARRAY_START_ROW)
|
|
|
|
|
@@ -658,6 +688,8 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool StartObject() {
|
|
|
|
|
+ set_prefix_match();
|
|
|
|
|
+
|
|
|
|
|
json_element_state.push_back(OBJECT_STATE);
|
|
|
|
|
m_current_depth++;
|
|
|
|
|
if(key_path.size()){
|
|
|
|
|
@@ -690,6 +722,8 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool StartArray() {
|
|
|
|
|
+ set_prefix_match();
|
|
|
|
|
+
|
|
|
|
|
json_element_state.push_back(ARRAY_STATE);
|
|
|
|
|
m_current_depth++;
|
|
|
|
|
if(key_path.size()){
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_oper.h ceph-19.2.2/src/s3select/include/s3select_oper.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select_oper.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select_oper.h 2025-07-07 10:22:00.000000000 -0400
|
|
|
|
|
@@ -592,9 +592,9 @@
|
|
|
|
|
|
|
|
|
|
value_En_t _type() const { return type; }
|
|
|
|
|
|
|
|
|
|
- void set_json_key_path(std::vector<std::string>& key_path)
|
|
|
|
|
+ void set_json_key_path(const std::vector<std::string>& key_path)
|
|
|
|
|
{
|
|
|
|
|
- m_json_key = key_path;
|
|
|
|
|
+ m_json_key = key_path;//TODO not efficient
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const char* to_string() //TODO very intensive , must improve this
|
|
|
|
|
@@ -693,9 +693,10 @@
|
|
|
|
|
m_str_value = o.m_str_value;
|
|
|
|
|
__val.str = m_str_value.data();
|
|
|
|
|
}
|
|
|
|
|
- else if(o.__val.str)
|
|
|
|
|
+ else if(o.__val.str)//it is done upon using the set_string_nocopy
|
|
|
|
|
{
|
|
|
|
|
- __val.str = o.__val.str;
|
|
|
|
|
+ m_str_value.assign(o.__val.str);//need to create a copy for this
|
|
|
|
|
+ __val.str = m_str_value.data();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
@@ -717,9 +718,10 @@
|
|
|
|
|
m_str_value = o.m_str_value;
|
|
|
|
|
__val.str = m_str_value.data();
|
|
|
|
|
}
|
|
|
|
|
- else if(o.__val.str)
|
|
|
|
|
+ else if(o.__val.str)//it is done upon using the set_string_nocopy
|
|
|
|
|
{
|
|
|
|
|
- __val.str = o.__val.str;
|
|
|
|
|
+ m_str_value.assign(o.__val.str);
|
|
|
|
|
+ __val.str = m_str_value.data();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
@@ -1137,6 +1139,54 @@
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
+/////////////////////////
|
|
|
|
|
+#include <unordered_map>
|
|
|
|
|
+class json_key_value_store
|
|
|
|
|
+{
|
|
|
|
|
+//hash function for std::vector<std::string>
|
|
|
|
|
+struct VectorStringHash {
|
|
|
|
|
+ std::size_t operator()(const std::vector<std::string>& vec) const {
|
|
|
|
|
+ std::size_t seed = 0;
|
|
|
|
|
+ for (const auto& str : vec) {
|
|
|
|
|
+ for (char c : str) {
|
|
|
|
|
+ // boost::hash_combine_size_t
|
|
|
|
|
+ seed ^= std::hash<char>()(c) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return seed;
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+//equality comparison function for std::vector<std::string>
|
|
|
|
|
+struct VectorStringEqual {
|
|
|
|
|
+ bool operator()(const std::vector<std::string>& lhs, const std::vector<std::string>& rhs) const {
|
|
|
|
|
+ return lhs == rhs;
|
|
|
|
|
+ }
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+public:
|
|
|
|
|
+
|
|
|
|
|
+//NOTE: using unordered_map(and also std::map) means that output result is NOT as the input order (TODO consider using an order column)
|
|
|
|
|
+//order means, order of keys, as appear in the input. order has no affect on the correctness of results.
|
|
|
|
|
+//the current flow, for identicals keys, save the first key/value and ignore all the rest.
|
|
|
|
|
+//
|
|
|
|
|
+//this type of container avoid huge memeory consumption upon a very big row(e.g. { "root" : [ long-and-nested-objects ] } )
|
|
|
|
|
+//a 'select * from s3object[*];' has to scan the entire "root" object before returing a result,
|
|
|
|
|
+//the star-operation must upload all keys-values for that row.
|
|
|
|
|
+//without a unique-key machanism it will consume large amount of memory(entire input JSON object).
|
|
|
|
|
+//
|
|
|
|
|
+//NOTE: upon huge JSON object containing unique-keys(random names), it will cause high memory consumption.
|
|
|
|
|
+//TODO:consider using container size limitation.
|
|
|
|
|
+typedef std::unordered_map<std::vector<std::string>, value, VectorStringHash, VectorStringEqual> _unorder_map_t;
|
|
|
|
|
+
|
|
|
|
|
+_unorder_map_t json_key_value_store;
|
|
|
|
|
+
|
|
|
|
|
+void insert(const std::vector<std::string>& key,value v)
|
|
|
|
|
+{
|
|
|
|
|
+ json_key_value_store.emplace( std::pair<std::vector<std::string>,value>(key,v) );
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
+};
|
|
|
|
|
|
|
|
|
|
class scratch_area
|
|
|
|
|
{
|
|
|
|
|
@@ -1155,8 +1205,8 @@
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
typedef std::pair<std::vector<std::string>,value> json_key_value_t;
|
|
|
|
|
- typedef std::vector< json_key_value_t > json_star_op_cont_t;
|
|
|
|
|
- json_star_op_cont_t m_json_star_operation;
|
|
|
|
|
+
|
|
|
|
|
+ json_key_value_store m_json_star_operation;
|
|
|
|
|
|
|
|
|
|
scratch_area():m_upper_bound(-1),parquet_type(false),buff_loc(0),max_json_idx(-1)
|
|
|
|
|
{
|
|
|
|
|
@@ -1168,14 +1218,20 @@
|
|
|
|
|
delete m_schema_values;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- json_star_op_cont_t* get_star_operation_cont()
|
|
|
|
|
+ json_key_value_store::_unorder_map_t* get_star_operation_cont()
|
|
|
|
|
+ {
|
|
|
|
|
+ return &m_json_star_operation.json_key_value_store;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ void json_push_key_value_per_star_operation(json_key_value_t& key_value)
|
|
|
|
|
{
|
|
|
|
|
- return &m_json_star_operation;
|
|
|
|
|
+ //identical keys(and thier values) are not saved.
|
|
|
|
|
+ m_json_star_operation.insert(key_value.first,key_value.second);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void clear_data()
|
|
|
|
|
{
|
|
|
|
|
- m_json_star_operation.clear();
|
|
|
|
|
+ m_json_star_operation.json_key_value_store.clear();
|
|
|
|
|
for(int i=0;i<=max_json_idx;i++)
|
|
|
|
|
{
|
|
|
|
|
(*m_schema_values)[i].setnull();
|
|
|
|
|
@@ -1857,8 +1913,8 @@
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
m_scratch->get_column_value(column_pos,var_value);
|
|
|
|
|
- //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL
|
|
|
|
|
- if (var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){
|
|
|
|
|
+ //in the case of successive column-delimiter {1,some_data,,3}=> third column is NULL(CSV data source)
|
|
|
|
|
+ if (!is_json_statement() && var_value.is_string() && (var_value.str()== 0 || (var_value.str() && *var_value.str()==0))){
|
|
|
|
|
var_value.setnull();//TODO is it correct for Parquet
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h 2025-07-07 13:08:00.747275733 -0400
|
|
|
|
|
@@ -434,7 +434,7 @@
|
|
|
|
|
#if ARROW_VERSION_MAJOR < 9
|
|
|
|
|
fd_ = -1;
|
|
|
|
|
#else
|
|
|
|
|
- fd_.Close();
|
|
|
|
|
+ return fd_.Close();
|
|
|
|
|
#endif
|
|
|
|
|
//RETURN_NOT_OK(::arrow::internal::FileClose(fd));
|
|
|
|
|
}
|
|
|
|
|
@@ -589,17 +589,12 @@
|
|
|
|
|
public:
|
|
|
|
|
|
|
|
|
|
~ReadableFileImpl()
|
|
|
|
|
- {
|
|
|
|
|
- if(IMPL != nullptr)
|
|
|
|
|
- {
|
|
|
|
|
- delete IMPL;
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
+ {}
|
|
|
|
|
|
|
|
|
|
#ifdef CEPH_USE_FS
|
|
|
|
|
- explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=new OSFile();}
|
|
|
|
|
+ explicit ReadableFileImpl(MemoryPool* pool) : pool_(pool) {IMPL=std::make_unique<OSFile>();}
|
|
|
|
|
#endif
|
|
|
|
|
- explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=new RGWimpl(rgw);}
|
|
|
|
|
+ explicit ReadableFileImpl(MemoryPool* pool,s3selectEngine::rgw_s3select_api* rgw) : pool_(pool) {IMPL=std::make_unique<RGWimpl>(rgw);}
|
|
|
|
|
|
|
|
|
|
Status Open(const std::string& path) { return IMPL->OpenReadable(path); }
|
|
|
|
|
|
|
|
|
|
@@ -649,7 +644,7 @@
|
|
|
|
|
return Status::OK();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- ObjectInterface *IMPL;//TODO to declare in ObjectInterface
|
|
|
|
|
+ std::unique_ptr<ObjectInterface> IMPL;
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
|
|
|
|
|
@@ -752,6 +747,30 @@
|
|
|
|
|
std::unique_ptr<Contents> contents_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
+#define RGW_default_buffer_size 1024*1024*1024
|
|
|
|
|
+class S3select_Config {
|
|
|
|
|
+public:
|
|
|
|
|
+ static S3select_Config& getInstance() {
|
|
|
|
|
+ static S3select_Config instance;
|
|
|
|
|
+ return instance;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ void set_s3select_reader_properties(uint64_t value) { this->m_reader_properties = value; }
|
|
|
|
|
+ uint64_t get_s3select_reader_properties() const { return m_reader_properties; }
|
|
|
|
|
+
|
|
|
|
|
+private:
|
|
|
|
|
+ uint64_t m_reader_properties;
|
|
|
|
|
+ S3select_Config() : m_reader_properties(RGW_default_buffer_size) {} // Private constructor
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+ReaderProperties s3select_reader_properties() {
|
|
|
|
|
+
|
|
|
|
|
+ static ReaderProperties default_reader_properties;
|
|
|
|
|
+ default_reader_properties.enable_buffered_stream();
|
|
|
|
|
+ default_reader_properties.set_buffer_size(S3select_Config::getInstance().get_s3select_reader_properties());
|
|
|
|
|
+ return default_reader_properties;
|
|
|
|
|
+}
|
|
|
|
|
+
|
|
|
|
|
class PARQUET_EXPORT ParquetFileReader {
|
|
|
|
|
public:
|
|
|
|
|
// Declare a virtual class 'Contents' to aid dependency injection and more
|
|
|
|
|
@@ -760,7 +779,7 @@
|
|
|
|
|
struct PARQUET_EXPORT Contents {
|
|
|
|
|
static std::unique_ptr<Contents> Open(
|
|
|
|
|
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
|
|
|
|
- const ReaderProperties& props = default_reader_properties(),
|
|
|
|
|
+ const ReaderProperties& props = s3select_reader_properties(),
|
|
|
|
|
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
|
|
|
|
|
|
|
|
|
virtual ~Contents() = default;
|
|
|
|
|
@@ -781,21 +800,21 @@
|
|
|
|
|
ARROW_DEPRECATED("Use arrow::io::RandomAccessFile version")
|
|
|
|
|
static std::unique_ptr<ParquetFileReader> Open(
|
|
|
|
|
std::unique_ptr<RandomAccessSource> source,
|
|
|
|
|
- const ReaderProperties& props = default_reader_properties(),
|
|
|
|
|
+ const ReaderProperties& props = s3select_reader_properties(),
|
|
|
|
|
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
|
|
|
|
|
|
|
|
|
// Create a file reader instance from an Arrow file object. Thread-safety is
|
|
|
|
|
// the responsibility of the file implementation
|
|
|
|
|
static std::unique_ptr<ParquetFileReader> Open(
|
|
|
|
|
std::shared_ptr<::arrow::io::RandomAccessFile> source,
|
|
|
|
|
- const ReaderProperties& props = default_reader_properties(),
|
|
|
|
|
+ const ReaderProperties& props = s3select_reader_properties(),
|
|
|
|
|
std::shared_ptr<FileMetaData> metadata = NULLPTR);
|
|
|
|
|
|
|
|
|
|
// API Convenience to open a serialized Parquet file on disk, using Arrow IO
|
|
|
|
|
// interfaces.
|
|
|
|
|
static std::unique_ptr<ParquetFileReader> OpenFile(
|
|
|
|
|
const std::string& path,s3selectEngine::rgw_s3select_api* rgw, bool memory_map = true,
|
|
|
|
|
- const ReaderProperties& props = default_reader_properties(),
|
|
|
|
|
+ const ReaderProperties& props = s3select_reader_properties(),
|
|
|
|
|
std::shared_ptr<FileMetaData> metadata = NULLPTR
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
@@ -983,6 +1002,7 @@
|
|
|
|
|
throw ParquetException("Encrypted files cannot contain more than 32767 row groups");
|
|
|
|
|
}
|
|
|
|
@@ -1611,15 +796,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
@@ -1039,7 +1078,7 @@
|
|
|
|
|
class SerializedFile : public ParquetFileReader::Contents {
|
|
|
|
|
public:
|
|
|
|
|
SerializedFile(std::shared_ptr<ArrowInputFile> source,
|
|
|
|
|
- const ReaderProperties& props = default_reader_properties())
|
|
|
|
|
+ const ReaderProperties& props = s3select_reader_properties())
|
|
|
|
|
: source_(std::move(source)), properties_(props) {
|
|
|
|
|
PARQUET_ASSIGN_OR_THROW(source_size_, source_->GetSize());
|
|
|
|
|
}
|
|
|
|
|
@@ -1052,7 +1091,9 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -1648,21 +824,6 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
|
|
|
|
|
|
|
|
|
|
int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len;
|
|
|
|
|
uint32_t metadata_len = footer_len - crypto_metadata_len;
|
|
|
|
|
@@ -1244,9 +1293,13 @@
|
|
|
|
|
std::to_string(metadata_buffer->size()) + " bytes)");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
+#if ARROW_VERSION_MAJOR > 9
|
|
|
|
|
+ file_metadata_ =
|
|
|
|
|
+ FileMetaData::Make(metadata_buffer->data(), &metadata_len, s3select_reader_properties(), file_decryptor_);
|
|
|
|
|
+#else
|
|
|
|
|
file_metadata_ =
|
|
|
|
|
FileMetaData::Make(metadata_buffer->data(), &metadata_len, file_decryptor_);
|
|
|
|
|
- //FileMetaData::Make(metadata_buffer->data(), &metadata_len, default_reader_properties(), file_decryptor_); //version>9
|
|
|
|
|
+#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void SerializedFile::ParseMetaDataOfEncryptedFileWithPlaintextFooter(
|
|
|
|
|
@@ -1259,9 +1312,18 @@
|
|
|
|
|
EncryptionAlgorithm algo = file_metadata_->encryption_algorithm();
|
|
|
|
|
// Handle AAD prefix
|
|
|
|
@@ -1682,57 +843,3 @@ diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.
|
|
|
|
|
// set the InternalFileDecryptor in the metadata as well, as it's used
|
|
|
|
|
// for signature verification and for ColumnChunkMetaData creation.
|
|
|
|
|
#if GAL_set_file_decryptor_declare_private
|
|
|
|
|
@@ -1584,7 +1646,7 @@
|
|
|
|
|
int m_num_row_groups;
|
|
|
|
|
std::shared_ptr<parquet::FileMetaData> m_file_metadata;
|
|
|
|
|
std::unique_ptr<parquet::ceph::ParquetFileReader> m_parquet_reader;
|
|
|
|
|
- std::vector<column_reader_wrap*> m_column_readers;
|
|
|
|
|
+ std::vector<std::shared_ptr<column_reader_wrap>> m_column_readers;
|
|
|
|
|
s3selectEngine::rgw_s3select_api* m_rgw_s3select_api;
|
|
|
|
|
|
|
|
|
|
public:
|
|
|
|
|
@@ -1603,12 +1665,7 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
~parquet_file_parser()
|
|
|
|
|
- {
|
|
|
|
|
- for(auto r : m_column_readers)
|
|
|
|
|
- {
|
|
|
|
|
- delete r;
|
|
|
|
|
- }
|
|
|
|
|
- }
|
|
|
|
|
+ {}
|
|
|
|
|
|
|
|
|
|
int load_meta_data()
|
|
|
|
|
{
|
|
|
|
|
@@ -1658,7 +1715,7 @@
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
- m_column_readers.push_back(new column_reader_wrap(m_parquet_reader,i));
|
|
|
|
|
+ m_column_readers.push_back(std::make_shared<column_reader_wrap>(m_parquet_reader,i));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
--- ceph-19.2.2/src/rgw/rgw_s3select.cc.orig 2025-07-07 14:24:23.371796969 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/rgw/rgw_s3select.cc 2025-07-07 17:45:30.655554945 -0400
|
|
|
|
|
@@ -472,6 +472,7 @@
|
|
|
|
|
if (!m_s3_parquet_object.is_set()) {
|
|
|
|
|
//parsing the SQL statement.
|
|
|
|
|
s3select_syntax.parse_query(m_sql_query.c_str());
|
|
|
|
|
+ parquet_object::csv_definitions parquet;
|
|
|
|
|
|
|
|
|
|
m_s3_parquet_object.set_external_system_functions(fp_s3select_continue,
|
|
|
|
|
fp_s3select_result_format,
|
|
|
|
|
@@ -479,8 +480,9 @@
|
|
|
|
|
fp_debug_mesg);
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
- //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object
|
|
|
|
|
- m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api);
|
|
|
|
|
+ //at this stage the Parquet-processing requires for the meta-data that reside on Parquet object
|
|
|
|
|
+ // m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api);
|
|
|
|
|
+ m_s3_parquet_object.set_parquet_object(std::string("s3object"), &s3select_syntax, &m_rgw_api, parquet);
|
|
|
|
|
} catch(base_s3select_exception& e) {
|
|
|
|
|
ldpp_dout(this, 10) << "S3select: failed upon parquet-reader construction: " << e.what() << dendl;
|
|
|
|
|
fp_result_header_format(m_aws_response_handler.get_sql_result());
|
|
|
|
|