Compare commits

..

4 Commits

Author SHA1 Message Date
Kaleb S. KEITHLEY 5378ce3de9 ceph-17.2.7, Requires: fuse -> fuse3
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
2023-11-02 05:46:40 -04:00
Kaleb S. KEITHLEY 888704869d ceph-17.2.7 GA
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
2023-10-27 06:37:37 -04:00
Kaleb S. KEITHLEY 1926382ebc ceph-17.2.6, restore WITH_SYSTEM_BOOST=YES (f38 still has boost-1.78)
Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
2023-05-22 19:24:01 -04:00
Kaleb S. KEITHLEY 9d75cd4f29 ceph-17.2.6, enable IBT, SHSTK when building bundled boost, librgw.so
libceph-common.so

Signed-off-by: Kaleb S. KEITHLEY <kkeithle@redhat.com>
2023-05-01 07:56:03 -04:00
39 changed files with 3551 additions and 2406 deletions
+11
View File
@@ -0,0 +1,11 @@
--- ceph-16.1.0-43-g6b74fb5c/cmake/modules/Finduring.cmake.orig 2021-02-01 08:45:39.316108287 -0500
+++ ceph-16.1.0-43-g6b74fb5c/cmake/modules/Finduring.cmake 2021-02-01 08:45:59.813665378 -0500
@@ -5,7 +5,7 @@
# uring_FOUND - True if uring found.
find_path(URING_INCLUDE_DIR liburing.h)
-find_library(URING_LIBRARIES liburing.a liburing)
+find_library(URING_LIBRARIES liburing.so liburing)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(uring DEFAULT_MSG URING_LIBRARIES URING_INCLUDE_DIR)
+63
View File
@@ -0,0 +1,63 @@
--- ceph-16.2.7/src/include/buffer.h.orig 2022-01-17 12:17:19.193356237 -0500
+++ ceph-16.2.7/src/include/buffer.h 2022-01-17 12:17:58.599639592 -0500
@@ -38,6 +38,7 @@
# include <sys/mman.h>
#endif
+#include <memory>
#include <iosfwd>
#include <iomanip>
#include <list>
--- ceph-16.2.7/src/common/LogEntry.cc.orig 2022-01-17 13:52:10.799134159 -0500
+++ ceph-16.2.7/src/common/LogEntry.cc 2022-01-17 13:52:47.244469274 -0500
@@ -183,7 +183,7 @@
return "crit";
default:
ceph_abort();
- return 0;
+ return "";
}
}
--- ceph-16.2.7/src/test/librados/tier_cxx.cc.orig 2022-01-19 09:30:47.209459506 -0500
+++ ceph-16.2.7/src/test/librados/tier_cxx.cc 2022-01-19 10:02:47.783240298 -0500
@@ -114,7 +114,7 @@
#include "rgw/rgw_common.h"
void check_fp_oid_refcount(librados::IoCtx& ioctx, std::string foid, uint64_t count,
- std::string fp_algo = NULL)
+ std::string fp_algo = "")
{
bufferlist t;
int size = foid.length();
@@ -142,7 +142,7 @@
ASSERT_LE(count, refs.count());
}
-string get_fp_oid(string oid, std::string fp_algo = NULL)
+string get_fp_oid(string oid, std::string fp_algo = "")
{
if (fp_algo == "sha1") {
unsigned char fingerprint[CEPH_CRYPTO_SHA1_DIGESTSIZE + 1];
--- ceph-16.2.7/src/test/test_trans.cc.orig 2022-01-19 13:24:33.460008897 -0500
+++ ceph-16.2.7/src/test/test_trans.cc 2022-01-19 13:24:58.211554005 -0500
@@ -51,7 +51,7 @@
cout << "#dev " << filename << std::endl;
cout << "#mb " << mb << std::endl;
- ObjectStore *fs = new FileStore(cct.get(), filename, NULL);
+ ObjectStore *fs = new FileStore(cct.get(), filename, "");
if (fs->mount() < 0) {
cout << "mount failed" << std::endl;
return -1;
--- ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h.orig 2022-02-11 17:21:40.268627997 -0500
+++ ceph-17.0.0-10335-gfd206722/src/s3select/include/s3select_functions.h 2022-02-11 17:21:57.155325437 -0500
@@ -466,7 +466,7 @@
std::string print(int ident) override
{
- return std::string(0);
+ return std::string("");
}
void push_argument(base_statement* arg)
@@ -1,15 +1,24 @@
--- ceph-18.0.0-3078-gc4847bf8/src/rgw/driver/dbstore/CMakeLists.txt.orig 2023-05-10 08:23:50.000000000 -0400
+++ ceph-18.0.0-3078-gc4847bf8/src/rgw/driver/dbstore/CMakeLists.txt 2023-05-11 08:21:13.794152904 -0400
@@ -24,7 +24,7 @@
--- ceph-17.1.0/src/rgw/store/dbstore/sqlite/CMakeLists.txt.orig 2022-03-01 08:19:04.974902872 -0500
+++ ceph-17.1.0/src/rgw/store/dbstore/sqlite/CMakeLists.txt 2022-03-11 07:55:16.236261471 -0500
@@ -12,5 +12,5 @@
set(SQLITE_COMPILE_FLAGS "-DSQLITE_THREADSAFE=1")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SQLITE_COMPILE_FLAGS}")
-add_library(sqlite_db ${sqlite_db_srcs})
+add_library(sqlite_db STATIC ${sqlite_db_srcs})
target_link_libraries(sqlite_db sqlite3 dbstore_lib rgw_common)
--- ceph-17.1.0/src/rgw/store/dbstore/CMakeLists.txt.orig 2022-02-28 14:11:49.987077811 -0500
+++ ceph-17.1.0/src/rgw/store/dbstore/CMakeLists.txt 2022-03-11 08:40:13.409682698 -0500
@@ -16,7 +16,7 @@
dbstore_mgr.cc
)
-add_library(dbstore_lib ${dbstore_srcs})
+add_library(dbstore_lib STATIC ${dbstore_srcs})
target_include_directories(dbstore_lib
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw"
PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/store/rados"
@@ -49,6 +49,7 @@
target_include_directories(dbstore_lib PUBLIC "${CMAKE_SOURCE_DIR}/src/fmt/include")
target_include_directories(dbstore_lib PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw")
set(link_targets spawn)
@@ -38,6 +38,7 @@
# add pthread library
set (CMAKE_LINK_LIBRARIES ${CMAKE_LINK_LIBRARIES} pthread)
@@ -17,3 +26,12 @@
find_package(gtest QUIET)
if(WITH_TESTS)
@@ -47,7 +48,7 @@
endif()
include_directories(${CMAKE_INCLUDE_DIR})
-add_library(dbstore ${dbstore_mgr_srcs})
+add_library(dbstore STATIC ${dbstore_mgr_srcs})
target_link_libraries(dbstore ${CMAKE_LINK_LIBRARIES})
# testing purpose
@@ -0,0 +1,43 @@
--- ceph-17.2.0-359-gb2fe9ec8/cmake/modules/CheckCxxAtomic.cmake.orig 2022-06-03 08:45:32.341075140 -0400
+++ ceph-17.2.0-359-gb2fe9ec8/cmake/modules/CheckCxxAtomic.cmake 2022-06-03 08:46:47.195775813 -0400
@@ -10,8 +10,9 @@
check_cxx_source_compiles("
#include <atomic>
#include <cstdint>
+#include <cstddef>
-#if defined(__s390x__) || defined(__mips__)
+#if defined(__SIZEOF_INT128__)
// Boost needs 16-byte atomics for tagged pointers.
// These are implemented via inline instructions on the platform
// if 16-byte alignment can be proven, and are delegated to libatomic
@@ -21,13 +22,27 @@
// We specifically test access via an otherwise unknown pointer here
// to ensure we get the most complex case. If this access can be
// done without libatomic, then all accesses can be done.
-bool atomic16(std::atomic<unsigned __int128> *ptr)
+struct tagged_ptr {
+ int* ptr;
+ std::size_t tag;
+};
+
+void atomic16(std::atomic<tagged_ptr> *ptr) __attribute__ ((used));
+void atomic16(std::atomic<tagged_ptr> *ptr)
{
- return *ptr != 0;
+ tagged_ptr p{nullptr, 1};
+ ptr->store(p);
+ tagged_ptr f = ptr->load();
+ tagged_ptr new_tag{nullptr, 0};
+ ptr->compare_exchange_strong(f, new_tag);
}
#endif
int main() {
+#if defined(__SIZEOF_INT128__)
+ std::atomic<tagged_ptr> ptr;
+ atomic16(&ptr);
+#endif
std::atomic<uint8_t> w1;
std::atomic<uint16_t> w2;
std::atomic<uint32_t> w4;
@@ -0,0 +1,11 @@
--- ceph-17.1.0-175-g086c8f84/src/arrow/cpp/cmake_modules/ThirdpartyToolchain.cmake.orig 2022-04-08 11:27:53.593570634 -0400
+++ ceph-17.1.0-175-g086c8f84/src/arrow/cpp/cmake_modules/ThirdpartyToolchain.cmake 2022-04-08 11:28:20.778087653 -0400
@@ -1991,7 +1991,7 @@
if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQUAL "NONE"
))
- set(xsimd_SOURCE "BUNDLED")
+ set(xsimd_SOURCE "SYSTEM")
resolve_dependency(xsimd)
# TODO: Don't use global includes but rather target_include_directories
include_directories(SYSTEM ${XSIMD_INCLUDE_DIR})
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,28 @@
From fff72cd14c58d06774cbd0274e6144b42448af03 Mon Sep 17 00:00:00 2001
From: "Adam C. Emerson" <aemerson@redhat.com>
Date: Mon, 7 Mar 2022 18:54:30 -0500
Subject: [PATCH] mon: Replace deprecated use of format_to
The non-deprecated version takes an explicit OutputIterator.
Signed-off-by: Adam C. Emerson <aemerson@redhat.com>
---
src/mon/LogMonitor.cc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/mon/LogMonitor.cc b/src/mon/LogMonitor.cc
index 9103ddf7c5b..c196e8429fb 100644
--- a/src/mon/LogMonitor.cc
+++ b/src/mon/LogMonitor.cc
@@ -411,7 +411,7 @@ void LogMonitor::log_external(const LogEntry& le)
}
if (fd >= 0) {
- fmt::format_to(file_log_buffer, "{}\n", le);
+ fmt::format_to(std::back_inserter(file_log_buffer), "{}\n", le);
int err = safe_write(fd, file_log_buffer.data(), file_log_buffer.size());
file_log_buffer.clear();
if (err < 0) {
--
2.36.1
@@ -0,0 +1,218 @@
--- ceph-17.2.3/src/s3select/include/s3select_parquet_intrf.h.orig 2022-01-11 15:47:52.000000000 -0500
+++ ceph-17.2.3/src/s3select/include/s3select_parquet_intrf.h 2022-08-22 10:26:06.738082924 -0400
@@ -26,6 +26,14 @@
#include "internal_file_decryptor.h"
#include "encryption_internal.h"
+#if ARROW_VERSION_MAJOR < 9
+#define _ARROW_FD fd_
+#define _ARROW_FD_TYPE int
+#else
+#define _ARROW_FD fd_.fd()
+#define _ARROW_FD_TYPE arrow::internal::FileDescriptor
+#endif
+
/******************************************/
/******************************************/
class optional_yield;
@@ -164,7 +172,7 @@
std::mutex lock_;
// File descriptor
- int fd_;
+ _ARROW_FD_TYPE fd_;
FileMode::type mode_;
@@ -202,7 +210,7 @@
mode_ = write_only ? FileMode::WRITE : FileMode::READWRITE;
if (!truncate) {
- ARROW_ASSIGN_OR_RAISE(size_, ::arrow::internal::FileGetSize(fd_));
+ ARROW_ASSIGN_OR_RAISE(size_, ::arrow::internal::FileGetSize(_ARROW_FD));
} else {
size_ = 0;
}
@@ -222,7 +230,11 @@
RETURN_NOT_OK(SetFileName(fd));
is_open_ = true;
mode_ = FileMode::WRITE;
+ #if ARROW_VERSION_MAJOR < 9
fd_ = fd;
+ #else
+ fd_ = arrow::internal::FileDescriptor{fd};
+ #endif
return Status::OK();
}
@@ -230,7 +242,7 @@
RETURN_NOT_OK(SetFileName(path));
ARROW_ASSIGN_OR_RAISE(fd_, ::arrow::internal::FileOpenReadable(file_name_));
- ARROW_ASSIGN_OR_RAISE(size_, ::arrow::internal::FileGetSize(fd_));
+ ARROW_ASSIGN_OR_RAISE(size_, ::arrow::internal::FileGetSize(_ARROW_FD));
is_open_ = true;
mode_ = FileMode::READ;
@@ -242,7 +254,11 @@
RETURN_NOT_OK(SetFileName(fd));
is_open_ = true;
mode_ = FileMode::READ;
+ #if ARROW_VERSION_MAJOR < 9
fd_ = fd;
+ #else
+ fd_ = arrow::internal::FileDescriptor{fd};
+ #endif
return Status::OK();
}
@@ -258,9 +274,13 @@
// Even if closing fails, the fd will likely be closed (perhaps it's
// already closed).
is_open_ = false;
+ #if ARROW_VERSION_MAJOR < 9
int fd = fd_;
fd_ = -1;
RETURN_NOT_OK(::arrow::internal::FileClose(fd));
+ #else
+ RETURN_NOT_OK(fd_.Close());
+ #endif
}
return Status::OK();
}
@@ -268,7 +288,7 @@
Result<int64_t> Read(int64_t nbytes, void* out) override {
RETURN_NOT_OK(CheckClosed());
RETURN_NOT_OK(CheckPositioned());
- return ::arrow::internal::FileRead(fd_, reinterpret_cast<uint8_t*>(out), nbytes);
+ return ::arrow::internal::FileRead(_ARROW_FD, reinterpret_cast<uint8_t*>(out), nbytes);
}
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override {
@@ -277,7 +297,7 @@
// ReadAt() leaves the file position undefined, so require that we seek
// before calling Read() or Write().
need_seeking_.store(true);
- return ::arrow::internal::FileReadAt(fd_, reinterpret_cast<uint8_t*>(out), position,
+ return ::arrow::internal::FileReadAt(_ARROW_FD, reinterpret_cast<uint8_t*>(out), position,
nbytes);
}
@@ -286,7 +306,7 @@
if (pos < 0) {
return Status::Invalid("Invalid position");
}
- Status st = ::arrow::internal::FileSeek(fd_, pos);
+ Status st = ::arrow::internal::FileSeek(_ARROW_FD, pos);
if (st.ok()) {
need_seeking_.store(false);
}
@@ -295,7 +315,7 @@
Result<int64_t> Tell() const override {
RETURN_NOT_OK(CheckClosed());
- return ::arrow::internal::FileTell(fd_);
+ return ::arrow::internal::FileTell(_ARROW_FD);
}
Status Write(const void* data, int64_t length) override {
@@ -306,11 +326,11 @@
if (length < 0) {
return Status::IOError("Length must be non-negative");
}
- return ::arrow::internal::FileWrite(fd_, reinterpret_cast<const uint8_t*>(data),
+ return ::arrow::internal::FileWrite(_ARROW_FD, reinterpret_cast<const uint8_t*>(data),
length);
}
- int fd() const override { return fd_; }
+ int fd() const override { return _ARROW_FD; }
bool is_open() const override { return is_open_; }
@@ -345,7 +365,7 @@
std::mutex lock_;
// File descriptor
- int fd_;
+ _ARROW_FD_TYPE fd_;
FileMode::type mode_;
@@ -411,7 +431,11 @@
// already closed).
is_open_ = false;
//int fd = fd_;
+ #if ARROW_VERSION_MAJOR < 9
fd_ = -1;
+ #else
+ fd_.Close();
+ #endif
//RETURN_NOT_OK(::arrow::internal::FileClose(fd));
}
return Status::OK();
@@ -421,7 +445,7 @@
NOT_IMPLEMENT;
RETURN_NOT_OK(CheckClosed());
RETURN_NOT_OK(CheckPositioned());
- return ::arrow::internal::FileRead(fd_, reinterpret_cast<uint8_t*>(out), nbytes);
+ return ::arrow::internal::FileRead(_ARROW_FD, reinterpret_cast<uint8_t*>(out), nbytes);
}
Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) {
@@ -443,7 +467,7 @@
return Status::OK();
}
- int fd() const { return fd_; }
+ int fd() const { return _ARROW_FD; }
bool is_open() const { return is_open_; }
@@ -467,7 +491,7 @@
std::mutex lock_;
// File descriptor
- int fd_;
+ _ARROW_FD_TYPE fd_;
FileMode::type mode_;
@@ -609,7 +633,7 @@
for (const auto& range : ranges) {
RETURN_NOT_OK(internal::ValidateRange(range.offset, range.length));
#if defined(POSIX_FADV_WILLNEED)
- if (posix_fadvise(fd_, range.offset, range.length, POSIX_FADV_WILLNEED)) {
+ if (posix_fadvise(_ARROW_FD, range.offset, range.length, POSIX_FADV_WILLNEED)) {
return IOErrorFromErrno(errno, "posix_fadvise failed");
}
#elif defined(F_RDADVISE) // macOS, BSD?
@@ -617,7 +641,7 @@
off_t ra_offset;
int ra_count;
} radvisory{range.offset, static_cast<int>(range.length)};
- if (radvisory.ra_count > 0 && fcntl(fd_, F_RDADVISE, &radvisory) == -1) {
+ if (radvisory.ra_count > 0 && fcntl(_ARROW_FD, F_RDADVISE, &radvisory) == -1) {
return IOErrorFromErrno(errno, "fcntl(fd, F_RDADVISE, ...) failed");
}
#endif
@@ -970,6 +994,9 @@
CryptoContext ctx(col->has_dictionary_page(), row_group_ordinal_,
static_cast<int16_t>(i), meta_decryptor, data_decryptor);
return PageReader::Open(stream, col->num_values(), col->compression(),
+ #if ARROW_VERSION_MAJOR > 8
+ false,
+ #endif
properties_.memory_pool(), &ctx);
}
@@ -985,6 +1012,9 @@
CryptoContext ctx(col->has_dictionary_page(), row_group_ordinal_,
static_cast<int16_t>(i), meta_decryptor, data_decryptor);
return PageReader::Open(stream, col->num_values(), col->compression(),
+ #if ARROW_VERSION_MAJOR > 8
+ false,
+ #endif
properties_.memory_pool(), &ctx);
}
+51 -20
View File
@@ -1,6 +1,6 @@
--- ceph-18.1.2/src/common/subsys_types.h.orig 2023-06-27 15:59:59.000000000 -0400
+++ ceph-18.1.2/src/common/subsys_types.h 2023-07-04 19:36:55.941238973 -0400
@@ -54,7 +54,7 @@
--- ceph-17.2.5/src/common/subsys_types.h.orig 2023-01-17 05:29:55.711592884 -0500
+++ ceph-17.2.5/src/common/subsys_types.h 2023-01-17 05:31:05.759282197 -0500
@@ -53,7 +53,7 @@
#undef DEFAULT_SUBSYS
}
@@ -9,8 +9,28 @@
ceph_subsys_get_max_default_level(const std::size_t subidx) {
const auto item = ceph_subsys_get_as_array()[subidx];
return std::max(item.log_level, item.gather_level);
--- ceph-18.1.2/src/msg/async/crypto_onwire.h.orig 2023-06-27 15:59:59.000000000 -0400
+++ ceph-18.1.2/src/msg/async/crypto_onwire.h 2023-07-04 19:36:55.957238704 -0400
--- ceph-17.2.5/src/msg/async/compression_onwire.h.orig 2023-01-17 07:34:31.923701878 -0500
+++ ceph-17.2.5/src/msg/async/compression_onwire.h 2023-01-17 07:35:04.493093534 -0500
@@ -44,7 +44,7 @@
class TxHandler final : private Handler {
public:
- TxHandler(CephContext* const cct, CompressorRef compressor, int mode, std::uint64_t min_size)
+ TxHandler(CephContext* const cct, CompressorRef compressor, int mode, uint64_t min_size)
: Handler(cct, compressor),
m_min_size(min_size),
m_mode(static_cast<Compressor::CompressionMode>(mode))
@@ -97,7 +97,7 @@
static rxtx_t create_handler_pair(
CephContext* ctx,
const CompConnectionMeta& comp_meta,
- std::uint64_t compress_min_size);
+ uint64_t compress_min_size);
};
}
--- ceph-17.2.5/src/msg/async/crypto_onwire.h.orig 2023-01-17 07:35:35.535513714 -0500
+++ ceph-17.2.5/src/msg/async/crypto_onwire.h 2023-01-17 07:35:46.578307452 -0500
@@ -95,7 +95,7 @@
// Transmitter can append extra bytes of ciphertext at the -final step.
// This method return how much was added, and thus let client translate
@@ -20,23 +40,34 @@
// Instance of RxHandler must be reset before doing any decrypt-update
// step. This applies also to situation when decrypt-final was already
--- ceph-18.1.2/src/rocksdb/table/block_based/data_block_hash_index.h.orig 2023-05-24 15:55:23.000000000 -0400
+++ ceph-18.1.2/src/rocksdb/table/block_based/data_block_hash_index.h 2023-07-04 19:36:55.971238469 -0400
@@ -8,6 +8,7 @@
#include <cstdint>
--- ceph-17.2.5/src/test/librados/op_speed.cc.orig 2023-01-17 08:57:37.078531022 -0500
+++ ceph-17.2.5/src/test/librados/op_speed.cc 2023-01-17 08:57:58.259139439 -0500
@@ -9,7 +9,7 @@
for (int i = 0; i < to_create; ++i) {
librados::ObjectReadOperation op;
bufferlist bl;
- std::uint64_t sz;
+ uint64_t sz;
struct timespec tm;
std::map<std::string, ceph::buffer::list> xattrs;
std::map<std::string, ceph::buffer::list> omap;
--- ceph-17.2.5/src/test/mon/test_log_rss_usage.cc.orig 2023-01-17 10:14:37.552820230 -0500
+++ ceph-17.2.5/src/test/mon/test_log_rss_usage.cc 2023-01-17 10:15:12.319202506 -0500
@@ -5,6 +5,7 @@
#include <string>
#include <vector>
+#include <cstdint>
#include "rocksdb/slice.h"
--- ceph-18.1.2/src/rocksdb/util/string_util.h.orig 2023-05-24 15:55:23.000000000 -0400
+++ ceph-18.1.2/src/rocksdb/util/string_util.h 2023-07-04 19:36:55.991238133 -0400
@@ -11,6 +11,7 @@
#include <string>
#include <unordered_map>
#include <vector>
#include <iostream>
#include <fstream>
+#include <cstdint>
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
--- ceph-17.2.5/src/rocksdb/db/compaction/compaction_iteration_stats.h.orig 2023-01-26 17:05:20.605333926 -0500
+++ ceph-17.2.5/src/rocksdb/db/compaction/compaction_iteration_stats.h 2023-01-26 17:05:46.376880846 -0500
@@ -6,6 +6,7 @@
#pragma once
#include "rocksdb/rocksdb_namespace.h"
+#include <cstdint>
struct CompactionIterationStats {
// Compaction statistics
@@ -0,0 +1,42 @@
From 73218e291ca68a927965bdffa7d43d0fc62c2718 Mon Sep 17 00:00:00 2001
From: Ondrej Mosnacek <omosnace@redhat.com>
Date: Wed, 27 Jul 2022 17:14:25 +0200
Subject: [PATCH] selinux: prepare for anon inode controls enablement
We plan to start labeling anon inodes (userfaultfd and io_uring file
descriptors) properly in selinux-policy, which means that domains using
these will need new rules.
See: https://github.com/fedora-selinux/selinux-policy/pull/1351
Since ceph may optionally use io_uring, this patch adds the necessary
interface call to its policy to avoid a regression. As the new interface
call is put under a conditional, the policy package will be buildable
against selinux-policy with or without the above PR merged, but it will
need to be rebuilt against the updated selinux-policy to actually pick
up the new rules.
I tested this on a minimal ceph cluster with 'bdev_ioring = true' added
to ceph.conf. I got io_uring denials without this patch + with
selinux-policy with PR#1351 and no denials with ceph rebuilt with this
patch.
Signed-off-by: Ondrej Mosnacek <omosnace@redhat.com>
---
selinux/ceph.te | 3 +++
1 file changed, 3 insertions(+)
diff --git a/selinux/ceph.te b/selinux/ceph.te
index 77d35d9714b60..729bce1fc8589 100644
--- a/selinux/ceph.te
+++ b/selinux/ceph.te
@@ -75,6 +75,9 @@ manage_lnk_files_pattern(ceph_t, ceph_var_run_t, ceph_var_run_t)
kernel_read_system_state(ceph_t)
kernel_read_network_state(ceph_t)
+ifdef(`kernel_io_uring_use',`
+ kernel_io_uring_use(ceph_t)
+')
allow ceph_t kernel_t:system module_request;
corenet_all_recvfrom_unlabeled(ceph_t)
@@ -0,0 +1,99 @@
--- ceph-17.2.5/src/boost/libs/python/src/object/enum.cpp.orig 2023-02-23 08:45:36.498595122 -0500
+++ ceph-17.2.5/src/boost/libs/python/src/object/enum.cpp 2023-02-23 08:46:11.277990890 -0500
@@ -153,7 +153,7 @@
{
if (enum_type_object.tp_dict == 0)
{
- Py_TYPE(&enum_type_object) = incref(&PyType_Type);
+ Py_SET_TYPE(&enum_type_object, incref(&PyType_Type));
#if PY_VERSION_HEX >= 0x03000000
enum_type_object.tp_base = &PyLong_Type;
#else
--- ceph-17.2.5/src/boost/libs/python/src/object/function.cpp.orig 2023-02-23 08:44:19.995920877 -0500
+++ ceph-17.2.5/src/boost/libs/python/src/object/function.cpp 2023-02-23 08:45:26.426770100 -0500
@@ -107,7 +107,7 @@
PyObject* p = this;
if (Py_TYPE(&function_type) == 0)
{
- Py_TYPE(&function_type) = &PyType_Type;
+ Py_SET_TYPE(&function_type, &PyType_Type);
::PyType_Ready(&function_type);
}
--- ceph-17.2.5/src/boost/libs/python/src/object/life_support.cpp.orig 2023-02-23 08:43:37.511650115 -0500
+++ ceph-17.2.5/src/boost/libs/python/src/object/life_support.cpp 2023-02-23 08:44:10.225088588 -0500
@@ -93,7 +93,7 @@
if (Py_TYPE(&life_support_type) == 0)
{
- Py_TYPE(&life_support_type) = &PyType_Type;
+ Py_SET_TYPE(&life_support_type, &PyType_Type);
PyType_Ready(&life_support_type);
}
--- ceph-17.2.5/src/boost/libs/python/src/object/class.cpp.orig 2023-02-23 08:46:22.394797757 -0500
+++ ceph-17.2.5/src/boost/libs/python/src/object/class.cpp 2023-02-23 10:54:56.016527900 -0500
@@ -21,6 +21,7 @@
#include <boost/python/dict.hpp>
#include <boost/python/str.hpp>
#include <boost/python/ssize_t.hpp>
+#include <boost/align/detail/align.hpp>
#include <functional>
#include <vector>
#include <cstddef>
@@ -208,7 +209,7 @@
{
if (static_data_object.tp_dict == 0)
{
- Py_TYPE(&static_data_object) = &PyType_Type;
+ Py_SET_TYPE(&static_data_object, &PyType_Type);
static_data_object.tp_base = &PyProperty_Type;
if (PyType_Ready(&static_data_object))
return 0;
@@ -316,7 +317,7 @@
{
if (class_metatype_object.tp_dict == 0)
{
- Py_TYPE(&class_metatype_object) = &PyType_Type;
+ Py_SET_TYPE(&class_metatype_object, &PyType_Type);
class_metatype_object.tp_base = &PyType_Type;
if (PyType_Ready(&class_metatype_object))
return type_handle();
@@ -374,12 +375,7 @@
// like, so we'll store the total size of the object
// there. A negative number indicates that the extra
// instance memory is not yet allocated to any holders.
-#if PY_VERSION_HEX >= 0x02060000
- Py_SIZE(result) =
-#else
- result->ob_size =
-#endif
- -(static_cast<int>(offsetof(instance<>,storage) + instance_size));
+ Py_SET_SIZE(result,-static_cast<int>(offsetof(instance<>,storage) + instance_size));
}
return (PyObject*)result;
}
@@ -470,7 +466,7 @@
{
if (class_type_object.tp_dict == 0)
{
- Py_TYPE(&class_type_object) = incref(class_metatype().get());
+ Py_SET_TYPE(&class_type_object, incref(class_metatype().get()));
class_type_object.tp_base = &PyBaseObject_Type;
if (PyType_Ready(&class_type_object))
return type_handle();
@@ -738,8 +734,13 @@
// holder_offset should at least point into the variable-sized part
assert(holder_offset >= offsetof(objects::instance<>,storage));
+ size_t allocated = holder_size + 8;
+ void* storage = (char*)self + holder_offset;
+ void* aligned_storage = ::boost::alignment::align(8, holder_size, storage, allocated);
+
// Record the fact that the storage is occupied, noting where it starts
- Py_SIZE(self) = holder_offset;
+ const size_t offset = reinterpret_cast<uintptr_t>(aligned_storage) - reinterpret_cast<uintptr_t>(storage) + holder_offset;
+ Py_SET_SIZE(self, offset);
return (char*)self + holder_offset;
}
else
+15
View File
@@ -0,0 +1,15 @@
--- ceph-17.2.6/cmake/modules/BuildBoost.cmake.orig 2023-04-27 14:00:28.239524778 -0400
+++ ceph-17.2.6/cmake/modules/BuildBoost.cmake 2023-04-28 07:49:59.743342207 -0400
@@ -63,7 +63,11 @@
else()
list(APPEND boost_features "address-model=32")
endif()
- set(BOOST_CXXFLAGS "-fPIC -w") # check on arm, etc <---XXX
+ if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+ set(BOOST_CXXFLAGS "-fPIC -w -fcf-protection") # check on arm, etc <---XXX
+ else()
+ set(BOOST_CXXFLAGS "-fPIC -w") # check on arm, etc <---XXX
+ endif()
list(APPEND boost_features "cxxflags=${BOOST_CXXFLAGS}")
set(boost_with_libs)
-17
View File
@@ -1,17 +0,0 @@
--- ceph-18.0.0-2726-g7cea3740/src/rgw/rgw_amqp.cc.orig 2023-03-14 18:22:35.636864260 -0400
+++ ceph-18.0.0-2726-g7cea3740/src/rgw/rgw_amqp.cc 2023-03-14 18:24:36.362756771 -0400
@@ -2,10 +2,10 @@
// vim: ts=8 sw=2 smarttab ft=cpp
#include "rgw_amqp.h"
-#include <amqp.h>
-#include <amqp_ssl_socket.h>
-#include <amqp_tcp_socket.h>
-#include <amqp_framing.h>
+#include <rabbitmq-c/amqp.h>
+#include <rabbitmq-c/ssl_socket.h>
+#include <rabbitmq-c/tcp_socket.h>
+#include <rabbitmq-c/framing.h>
#include "include/ceph_assert.h"
#include <sstream>
#include <cstring>
-24
View File
@@ -1,24 +0,0 @@
--- ceph-18.0.0-2950-g1c931bc4/cmake/modules/BuildBoost.cmake.orig 2023-04-28 18:30:19.133064577 -0400
+++ ceph-18.0.0-2950-g1c931bc4/cmake/modules/BuildBoost.cmake 2023-04-28 18:31:55.290354383 -0400
@@ -104,12 +104,21 @@
set(user_config ${CMAKE_BINARY_DIR}/user-config.jam)
# edit the user-config.jam so b2 will be able to use the specified
# toolset and python
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+ file(WRITE ${user_config}
+ "using ${toolset}"
+ " : "
+ " : ${CMAKE_CXX_COMPILER}"
+ " : <compileflags>-fPIC <compileflags>-w <compileflags>-fcf-protection <compileflags>-Wno-everything"
+ " ;\n")
+else()
file(WRITE ${user_config}
"using ${toolset}"
" : "
" : ${CMAKE_CXX_COMPILER}"
" : <compileflags>-fPIC <compileflags>-w <compileflags>-Wno-everything"
" ;\n")
+endif()
if(with_python_version)
find_package(Python3 ${with_python_version} QUIET REQUIRED
COMPONENTS Development)
-24
View File
@@ -1,24 +0,0 @@
--- ceph-18.1.2/src/pybind/rbd/rbd.pyx.orig 2023-07-21 13:30:08.156825317 -0400
+++ ceph-18.1.2/src/pybind/rbd/rbd.pyx 2023-07-24 09:09:27.930137117 -0400
@@ -371,10 +371,10 @@
cdef rados_ioctx_t convert_ioctx(rados.Ioctx ioctx) except? NULL:
return <rados_ioctx_t>ioctx.io
-cdef int progress_callback(uint64_t offset, uint64_t total, void* ptr) with gil:
+cdef int progress_callback(uint64_t offset, uint64_t total, void* ptr) noexcept with gil:
return (<object>ptr)(offset, total)
-cdef int no_op_progress_callback(uint64_t offset, uint64_t total, void* ptr):
+cdef int no_op_progress_callback(uint64_t offset, uint64_t total, void* ptr) noexcept:
return 0
def cstr(val, name, encoding="utf-8", opt=False):
@@ -426,7 +426,7 @@
cdef class Completion
-cdef void __aio_complete_cb(rbd_completion_t completion, void *args) with gil:
+cdef void __aio_complete_cb(rbd_completion_t completion, void *args) noexcept with gil:
"""
Callback to oncomplete() for asynchronous operations
"""
-18
View File
@@ -1,18 +0,0 @@
--- ceph/src/CMakeLists.txt.orig 2023-11-01 11:53:53.618167190 -0400
+++ ceph/src/CMakeLists.txt 2023-11-01 13:52:51.292643490 -0400
@@ -625,6 +625,7 @@
add_subdirectory(perfglue)
add_library(rados_snap_set_diff_obj OBJECT librados/snap_set_diff.cc)
+add_dependencies(rados_snap_set_diff_obj legacy-option-headers)
option(WITH_LIBRADOSSTRIPER "build with libradosstriper support" ON)
@@ -881,6 +882,7 @@
add_library(krbd STATIC krbd.cc
$<TARGET_OBJECTS:parse_secret_objs>)
target_link_libraries(krbd keyutils::keyutils)
+ add_dependencies(krbd legacy-option-headers)
endif()
add_subdirectory(librbd)
if(WITH_FUSE)
-41
View File
@@ -1,41 +0,0 @@
From f5d3e9146d9ceb12858bc94d60090bd851b1c088 Mon Sep 17 00:00:00 2001
From: Adam Emerson <aemerson@redhat.com>
Date: Wed, 20 Dec 2023 13:51:21 -0500
Subject: [PATCH] test/neorados: Use two fewer GTest internals
To get coroutine tests working, I used some internals. Two of these
are no longer available in newer versions. Since they can be
implemented with regular old C++ features, use those.
This fixes an FTBFS on Fedora Rawhide introduced in
`35231f7251ed70d8d817ee7e727d9763669d101f`.
Signed-off-by: Adam Emerson <aemerson@redhat.com>
---
src/test/neorados/common_tests.h | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/src/test/neorados/common_tests.h b/src/test/neorados/common_tests.h
index 9610f08f643f3..396432e417e84 100644
--- a/src/test/neorados/common_tests.h
+++ b/src/test/neorados/common_tests.h
@@ -391,10 +391,14 @@ class NeoRadosECTest : public NeoRadosTestBase {
public: \
GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() = default; \
~GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() override = default; \
- GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \
- test_name)); \
- GTEST_DISALLOW_MOVE_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \
- test_name)); \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)( \
+ const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)&) = delete; \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)& operator =( \
+ const GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)&) = delete; \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)( \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)&&) = delete; \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)& operator =( \
+ GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)&&) = delete; \
\
private: \
boost::asio::awaitable<void> CoTestBody() override; \
-19
View File
@@ -1,19 +0,0 @@
--- ceph-18.2.3/src/mgr/PyModule.cc.orig 2024-06-06 16:31:10.340731851 -0400
+++ ceph-18.2.3/src/mgr/PyModule.cc 2024-06-07 12:58:49.913571410 -0400
@@ -342,6 +342,16 @@
return ceph_module;
}
+// FIXME
+// Removed in Python 3.13 API, but kept for the stable ABI
+#ifdef __cplusplus
+extern "C" {
+#endif
+PyAPI_FUNC(void) PySys_SetPath(const wchar_t *);
+#ifdef __cplusplus
+}
+#endif
+
int PyModule::load(PyThreadState *pMainThreadState)
{
ceph_assert(pMainThreadState != nullptr);
-15
View File
@@ -1,15 +0,0 @@
--- ceph-19.1.0/src/common/crc32c_ppc_asm.S.orig 2024-07-10 10:55:06.255480790 -0400
+++ ceph-19.1.0/src/common/crc32c_ppc_asm.S 2024-07-10 12:53:42.844075490 -0400
@@ -54,11 +54,8 @@
/* byte reverse permute constant */
.octa 0x0F0E0D0C0B0A09080706050403020100
-#ifdef CRC32_CONSTANTS_HEADER
-#include CRC32_CONSTANTS_HEADER
-#else
+#define __ASSEMBLY__
#include "crc32c_ppc_constants.h"
-#endif
.text
-21
View File
@@ -1,21 +0,0 @@
--- ceph-19.1.0/src/cpp_redis/CMakeLists.txt.orig 2024-07-11 15:16:22.487082035 -0400
+++ ceph-19.1.0/src/cpp_redis/CMakeLists.txt 2024-07-11 15:17:57.226534456 -0400
@@ -174,6 +174,7 @@
set_target_properties(${PROJECT} PROPERTIES COMPILE_DEFINITIONS "__CPP_REDIS_USE_CUSTOM_TCP_CLIENT=${USE_CUSTOM_TCP_CLIENT}")
endif(USE_CUSTOM_TCP_CLIENT)
+install(TARGETS ${PROJECT} DESTINATION ${CMAKE_INSTALL_LIBDIR}/ceph)
###
# examples
--- ceph-19.1.0/src/cpp_redis/tacopie/CMakeLists.txt.orig 2024-01-30 21:47:59.000000000 -0500
+++ ceph-19.1.0/src/cpp_redis/tacopie/CMakeLists.txt 2024-07-12 07:37:23.584654640 -0400
@@ -162,6 +162,8 @@
ENDIF(SELECT_TIMEOUT)
+install(TARGETS ${PROJECT} DESTINATION ${CMAKE_INSTALL_LIBDIR}/ceph)
+
###
# examples
###
@@ -1,28 +0,0 @@
--- ceph-19.1.0/src/common/crc32c_ppc_fast_zero_asm.S.orig 2024-07-26 07:18:50.733484175 -0400
+++ ceph-19.1.0/src/common/crc32c_ppc_fast_zero_asm.S 2024-07-26 07:17:48.449531183 -0400
@@ -45,8 +45,10 @@
/* unsigned int barrett_reduction(unsigned long val) */
FUNC_START(barrett_reduction)
- lis r4,.constants@ha
- la r4,.constants@l(r4)
+/* lis r4,.constants@ha */
+/* la r4,.constants@l(r4) */
+ addis r4,r4,.constants@toc@ha
+ addi r4,r4,.constants@toc@l
li r5,16
vxor v1,v1,v1 /* zero v1 */
@@ -83,8 +85,10 @@
/* unsigned int barrett_reduction_reflected(unsigned long val) */
FUNC_START(barrett_reduction_reflected)
- lis r4,.bit_reflected_constants@ha
- la r4,.bit_reflected_constants@l(r4)
+/* lis r4,.bit_reflected_constants@ha */
+/* la r4,.bit_reflected_constants@l(r4) */
+ addis r4,r4,.bit_reflected_constants@toc@ha
+ addi r4,r4,.bit_reflected_constants@toc@l
li r5,16
vxor v1,v1,v1 /* zero v1 */
-55
View File
@@ -1,55 +0,0 @@
diff --git a/src/common/openssl_opts_handler.cc b/src/common/openssl_opts_handler.cc
index 81d0c4786..d9866f197 100644
--- a/src/common/openssl_opts_handler.cc
+++ b/src/common/openssl_opts_handler.cc
@@ -16,7 +16,10 @@
#include <openssl/bio.h>
#include <openssl/conf.h>
+#include <openssl/err.h>
+#ifndef OPENSSL_NO_ENGINE
#include <openssl/engine.h>
+#endif
#include <mutex>
#include <vector>
#include <algorithm>
@@ -116,11 +119,13 @@ void load_module(const string &engine_conf)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+#ifndef OPENSSL_NO_ENGINE
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wdeprecated-declarations"
ENGINE_load_builtin_engines();
#pragma clang diagnostic pop
#pragma GCC diagnostic pop
+#endif
if (CONF_modules_load(
conf, nullptr,
diff --git a/src/crypto/openssl/openssl_crypto_accel.cc b/src/crypto/openssl/openssl_crypto_accel.cc
index f99844a38..4944205dd 100644
--- a/src/crypto/openssl/openssl_crypto_accel.cc
+++ b/src/crypto/openssl/openssl_crypto_accel.cc
@@ -14,7 +14,6 @@
#include "crypto/openssl/openssl_crypto_accel.h"
#include <openssl/evp.h>
-#include <openssl/engine.h>
#include "common/debug.h"
// -----------------------------------------------------------------------------
diff --git a/src/jaegertracing/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/civetweb.c b/src/jaegertracing/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/civetweb.c
index fea9e6f98..c7eddb20b 100644
--- a/src/jaegertracing/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/civetweb.c
+++ b/src/jaegertracing/opentelemetry-cpp/third_party/prometheus-cpp/3rdparty/civetweb/src/civetweb.c
@@ -1744,7 +1744,9 @@ typedef struct SSL_CTX SSL_CTX;
#include <openssl/conf.h>
#include <openssl/crypto.h>
#include <openssl/dh.h>
+#if (OPENSSL_VERSION_NUMBER < 0x10100000L)
#include <openssl/engine.h>
+#endif
#include <openssl/err.h>
#include <openssl/opensslv.h>
#include <openssl/pem.h>
-16
View File
@@ -1,16 +0,0 @@
--- ceph-19.2.0/src/mds/CMakeLists.txt.orig 2024-12-20 17:15:47.632569537 -0500
+++ ceph-19.2.0/src/mds/CMakeLists.txt 2024-12-20 17:16:48.945481957 -0500
@@ -45,12 +45,11 @@
QuiesceDbManager.cc
QuiesceAgent.cc
MDSRankQuiesce.cc
- BoostUrlImpl.cc
${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
${CMAKE_SOURCE_DIR}/src/common/MemoryModel.cc
${CMAKE_SOURCE_DIR}/src/osdc/Journaler.cc
${CMAKE_SOURCE_DIR}/src/mgr/MDSPerfMetricTypes.cc)
add_library(mds STATIC ${mds_srcs})
target_link_libraries(mds PRIVATE
- heap_profiler cpu_profiler osdc ${LUA_LIBRARIES})
+ boost_url heap_profiler cpu_profiler osdc ${LUA_LIBRARIES})
target_include_directories(mds PRIVATE "${LUA_INCLUDE_DIR}")
@@ -1,20 +0,0 @@
--- ceph-19.2.0/src/rocksdb/db/blob/blob_file_meta.h.orig 2025-01-16 08:16:41.762758595 -0500
+++ ceph-19.2.0/src/rocksdb/db/blob/blob_file_meta.h 2025-01-16 08:17:37.172784779 -0500
@@ -10,6 +10,7 @@
#include <memory>
#include <string>
#include <unordered_set>
+#include <cstdint>
#include "rocksdb/rocksdb_namespace.h"
--- ceph-19.2.0/src/rocksdb/include/rocksdb/trace_record.h.orig 2025-01-16 09:04:19.196962397 -0500
+++ ceph-19.2.0/src/rocksdb/include/rocksdb/trace_record.h 2025-01-16 09:04:33.885701684 -0500
@@ -8,6 +8,7 @@
#include <memory>
#include <string>
#include <vector>
+#include <cstdint>
#include "rocksdb/rocksdb_namespace.h"
#include "rocksdb/slice.h"
-257
View File
@@ -1,257 +0,0 @@
--- ceph-19.2.0/src/rgw/driver/posix/zpp_bits.h.orig 2024-09-18 12:27:51.000000000 -0400
+++ ceph-19.2.0/src/rgw/driver/posix/zpp_bits.h 2025-01-16 14:58:48.471438398 -0500
@@ -2687,8 +2687,12 @@
{
using type = std::remove_cvref_t<decltype(container)>;
using value_type = typename type::value_type;
- constexpr auto is_const = std::is_const_v<
- std::remove_reference_t<decltype(container[0])>>;
+ constexpr auto is_const =
+ std::is_const_v<std::remove_reference_t<value_type>> ||
+ requires {
+ requires std::is_const_v<
+ std::remove_reference_t<decltype(container[0])>>;
+ };
if constexpr (!std::is_void_v<SizeType> &&
(requires(type container) { container.resize(1); } ||
@@ -2827,9 +2831,7 @@
if constexpr (requires { typename type::mapped_type; }) {
using value_type = std::pair<typename type::key_type,
typename type::mapped_type>;
- std::aligned_storage_t<sizeof(value_type),
- alignof(value_type)>
- storage;
+ alignas(value_type) std::byte storage[sizeof(value_type)];
auto object = access::placement_new<value_type>(
std::addressof(storage));
@@ -2843,9 +2845,7 @@
} else {
using value_type = typename type::value_type;
- std::aligned_storage_t<sizeof(value_type),
- alignof(value_type)>
- storage;
+ alignas(value_type) std::byte storage[sizeof(value_type)];
auto object = access::placement_new<value_type>(
std::addressof(storage));
@@ -2903,8 +2903,7 @@
return result;
}
} else {
- std::aligned_storage_t<sizeof(value_type), alignof(value_type)>
- storage;
+ alignas(value_type) std::byte storage[sizeof(value_type)];
auto object =
access::placement_new<value_type>(std::addressof(storage));
@@ -2947,9 +2946,7 @@
}
return serialize_one(*std::get_if<element_type>(&variant));
} else {
- std::aligned_storage_t<sizeof(element_type),
- alignof(element_type)>
- storage;
+ alignas(element_type) std::byte storage[sizeof(element_type)];
auto object = access::placement_new<element_type>(
std::addressof(storage));
@@ -2996,8 +2993,7 @@
return self.serialize_one(
*std::get_if<Types>(&variant));
} else {
- std::aligned_storage_t<sizeof(Types), alignof(Types)>
- storage;
+ alignas(Types) std::byte storage[sizeof(Types)];
auto object = access::placement_new<Types>(
std::addressof(storage));
@@ -3008,6 +3004,7 @@
return result;
}
variant = std::move(*object);
+ return errc{};
}
}...};
@@ -3307,7 +3304,7 @@
ZPP_BITS_INLINE constexpr static auto serialize(auto & serializer,
auto & self)
{
- return serializer.template serialize_one(self.variant, self.id);
+ return serializer.serialize_one(self.variant, self.id);
}
Variant & variant;
@@ -3448,13 +3445,14 @@
}
constexpr explicit value_or_errc(error_type error) :
- m_error(std::forward<decltype(error)>(error))
+ m_error(std::forward<decltype(error)>(error)),
+ m_failure(true)
{
}
constexpr value_or_errc(value_or_errc && other) noexcept
{
- if (other.is_value()) {
+ if (other.success()) {
if constexpr (!std::is_void_v<Type>) {
if constexpr (!std::is_reference_v<Type>) {
::new (std::addressof(m_return_value))
@@ -3752,16 +3750,16 @@
requires(decltype(in.remaining_data()) & data) {
(context.*Function)(data);
}) {
- struct _
+ struct guard
{
decltype(in) archive;
decltype(in.remaining_data()) data;
- constexpr ~_()
+ constexpr ~guard()
{
archive.position() += data.size();
}
- } _{in, in.remaining_data()};
- return (context.*Function)(_.data);
+ } guard{in, in.remaining_data()};
+ return (context.*Function)(guard.data);
} else {
return (context.*Function)();
}
@@ -3776,16 +3774,16 @@
requires(decltype(in.remaining_data()) & data) {
Function(data);
}) {
- struct _
+ struct guard
{
decltype(in) archive;
decltype(in.remaining_data()) data;
- constexpr ~_()
+ constexpr ~guard()
{
archive.position() += data.size();
}
- } _{in, in.remaining_data()};
- return Function(_.data);
+ } guard{in, in.remaining_data()};
+ return Function(guard.data);
} else {
return Function();
}
@@ -5029,9 +5027,7 @@
serialize use();
};
- std::aligned_storage_t<sizeof(value_type),
- alignof(value_type)>
- storage;
+ alignas(value_type) std::byte storage[sizeof(value_type)];
auto object =
access::placement_new<value_type>(std::addressof(storage));
@@ -5111,9 +5107,7 @@
return errc{};
}
} else {
- std::aligned_storage_t<sizeof(value_type),
- alignof(value_type)>
- storage;
+ alignas(value_type) std::byte storage[sizeof(value_type)];
auto object = access::placement_new<value_type>(
std::addressof(storage));
@@ -5382,47 +5376,47 @@
};
auto align = [](auto v, auto a) { return (v + (a - 1)) / a * a; };
- auto h0 = big_endian{0x6a09e667u};
- auto h1 = big_endian{0xbb67ae85u};
- auto h2 = big_endian{0x3c6ef372u};
- auto h3 = big_endian{0xa54ff53au};
- auto h4 = big_endian{0x510e527fu};
- auto h5 = big_endian{0x9b05688cu};
- auto h6 = big_endian{0x1f83d9abu};
- auto h7 = big_endian{0x5be0cd19u};
-
- std::array k{big_endian{0x428a2f98u}, big_endian{0x71374491u},
- big_endian{0xb5c0fbcfu}, big_endian{0xe9b5dba5u},
- big_endian{0x3956c25bu}, big_endian{0x59f111f1u},
- big_endian{0x923f82a4u}, big_endian{0xab1c5ed5u},
- big_endian{0xd807aa98u}, big_endian{0x12835b01u},
- big_endian{0x243185beu}, big_endian{0x550c7dc3u},
- big_endian{0x72be5d74u}, big_endian{0x80deb1feu},
- big_endian{0x9bdc06a7u}, big_endian{0xc19bf174u},
- big_endian{0xe49b69c1u}, big_endian{0xefbe4786u},
- big_endian{0x0fc19dc6u}, big_endian{0x240ca1ccu},
- big_endian{0x2de92c6fu}, big_endian{0x4a7484aau},
- big_endian{0x5cb0a9dcu}, big_endian{0x76f988dau},
- big_endian{0x983e5152u}, big_endian{0xa831c66du},
- big_endian{0xb00327c8u}, big_endian{0xbf597fc7u},
- big_endian{0xc6e00bf3u}, big_endian{0xd5a79147u},
- big_endian{0x06ca6351u}, big_endian{0x14292967u},
- big_endian{0x27b70a85u}, big_endian{0x2e1b2138u},
- big_endian{0x4d2c6dfcu}, big_endian{0x53380d13u},
- big_endian{0x650a7354u}, big_endian{0x766a0abbu},
- big_endian{0x81c2c92eu}, big_endian{0x92722c85u},
- big_endian{0xa2bfe8a1u}, big_endian{0xa81a664bu},
- big_endian{0xc24b8b70u}, big_endian{0xc76c51a3u},
- big_endian{0xd192e819u}, big_endian{0xd6990624u},
- big_endian{0xf40e3585u}, big_endian{0x106aa070u},
- big_endian{0x19a4c116u}, big_endian{0x1e376c08u},
- big_endian{0x2748774cu}, big_endian{0x34b0bcb5u},
- big_endian{0x391c0cb3u}, big_endian{0x4ed8aa4au},
- big_endian{0x5b9cca4fu}, big_endian{0x682e6ff3u},
- big_endian{0x748f82eeu}, big_endian{0x78a5636fu},
- big_endian{0x84c87814u}, big_endian{0x8cc70208u},
- big_endian{0x90befffau}, big_endian{0xa4506cebu},
- big_endian{0xbef9a3f7u}, big_endian{0xc67178f2u}};
+ auto h0 = big_endian{std::uint32_t{0x6a09e667u}};
+ auto h1 = big_endian{std::uint32_t{0xbb67ae85u}};
+ auto h2 = big_endian{std::uint32_t{0x3c6ef372u}};
+ auto h3 = big_endian{std::uint32_t{0xa54ff53au}};
+ auto h4 = big_endian{std::uint32_t{0x510e527fu}};
+ auto h5 = big_endian{std::uint32_t{0x9b05688cu}};
+ auto h6 = big_endian{std::uint32_t{0x1f83d9abu}};
+ auto h7 = big_endian{std::uint32_t{0x5be0cd19u}};
+
+ std::array k{big_endian{std::uint32_t{0x428a2f98u}}, big_endian{std::uint32_t{0x71374491u}},
+ big_endian{std::uint32_t{0xb5c0fbcfu}}, big_endian{std::uint32_t{0xe9b5dba5u}},
+ big_endian{std::uint32_t{0x3956c25bu}}, big_endian{std::uint32_t{0x59f111f1u}},
+ big_endian{std::uint32_t{0x923f82a4u}}, big_endian{std::uint32_t{0xab1c5ed5u}},
+ big_endian{std::uint32_t{0xd807aa98u}}, big_endian{std::uint32_t{0x12835b01u}},
+ big_endian{std::uint32_t{0x243185beu}}, big_endian{std::uint32_t{0x550c7dc3u}},
+ big_endian{std::uint32_t{0x72be5d74u}}, big_endian{std::uint32_t{0x80deb1feu}},
+ big_endian{std::uint32_t{0x9bdc06a7u}}, big_endian{std::uint32_t{0xc19bf174u}},
+ big_endian{std::uint32_t{0xe49b69c1u}}, big_endian{std::uint32_t{0xefbe4786u}},
+ big_endian{std::uint32_t{0x0fc19dc6u}}, big_endian{std::uint32_t{0x240ca1ccu}},
+ big_endian{std::uint32_t{0x2de92c6fu}}, big_endian{std::uint32_t{0x4a7484aau}},
+ big_endian{std::uint32_t{0x5cb0a9dcu}}, big_endian{std::uint32_t{0x76f988dau}},
+ big_endian{std::uint32_t{0x983e5152u}}, big_endian{std::uint32_t{0xa831c66du}},
+ big_endian{std::uint32_t{0xb00327c8u}}, big_endian{std::uint32_t{0xbf597fc7u}},
+ big_endian{std::uint32_t{0xc6e00bf3u}}, big_endian{std::uint32_t{0xd5a79147u}},
+ big_endian{std::uint32_t{0x06ca6351u}}, big_endian{std::uint32_t{0x14292967u}},
+ big_endian{std::uint32_t{0x27b70a85u}}, big_endian{std::uint32_t{0x2e1b2138u}},
+ big_endian{std::uint32_t{0x4d2c6dfcu}}, big_endian{std::uint32_t{0x53380d13u}},
+ big_endian{std::uint32_t{0x650a7354u}}, big_endian{std::uint32_t{0x766a0abbu}},
+ big_endian{std::uint32_t{0x81c2c92eu}}, big_endian{std::uint32_t{0x92722c85u}},
+ big_endian{std::uint32_t{0xa2bfe8a1u}}, big_endian{std::uint32_t{0xa81a664bu}},
+ big_endian{std::uint32_t{0xc24b8b70u}}, big_endian{std::uint32_t{0xc76c51a3u}},
+ big_endian{std::uint32_t{0xd192e819u}}, big_endian{std::uint32_t{0xd6990624u}},
+ big_endian{std::uint32_t{0xf40e3585u}}, big_endian{std::uint32_t{0x106aa070u}},
+ big_endian{std::uint32_t{0x19a4c116u}}, big_endian{std::uint32_t{0x1e376c08u}},
+ big_endian{std::uint32_t{0x2748774cu}}, big_endian{std::uint32_t{0x34b0bcb5u}},
+ big_endian{std::uint32_t{0x391c0cb3u}}, big_endian{std::uint32_t{0x4ed8aa4au}},
+ big_endian{std::uint32_t{0x5b9cca4fu}}, big_endian{std::uint32_t{0x682e6ff3u}},
+ big_endian{std::uint32_t{0x748f82eeu}}, big_endian{std::uint32_t{0x78a5636fu}},
+ big_endian{std::uint32_t{0x84c87814u}}, big_endian{std::uint32_t{0x8cc70208u}},
+ big_endian{std::uint32_t{0x90befffau}}, big_endian{std::uint32_t{0xa4506cebu}},
+ big_endian{std::uint32_t{0xbef9a3f7u}}, big_endian{std::uint32_t{0xc67178f2u}}};
constexpr auto original_message = to_bytes<Object>();
constexpr auto chunk_size = 512 / CHAR_BIT;
-42
View File
@@ -1,42 +0,0 @@
--- ceph-19.2.0/src/googletest/googletest/CMakeLists.txt.orig 2025-01-23 07:43:33.314970694 -0500
+++ ceph-19.2.0/src/googletest/googletest/CMakeLists.txt 2025-01-23 07:44:13.529250429 -0500
@@ -63,6 +63,8 @@
set_up_hermetic_build()
endif()
+option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)." OFF)
+
# These commands only run if this is the main project
if(CMAKE_PROJECT_NAME STREQUAL "gtest" OR CMAKE_PROJECT_NAME STREQUAL "googletest-distribution")
@@ -126,9 +128,9 @@
# are used for other targets, to ensure that gtest can be compiled by a user
# aggressive about warnings.
cxx_library(gtest "${cxx_strict}" src/gtest-all.cc)
-set_target_properties(gtest PROPERTIES VERSION ${GOOGLETEST_VERSION})
+#set_target_properties(gtest PROPERTIES VERSION ${GOOGLETEST_VERSION})
cxx_library(gtest_main "${cxx_strict}" src/gtest_main.cc)
-set_target_properties(gtest_main PROPERTIES VERSION ${GOOGLETEST_VERSION})
+#set_target_properties(gtest_main PROPERTIES VERSION ${GOOGLETEST_VERSION})
# If the CMake version supports it, attach header directory information
# to the targets for when we are part of a parent build (ie being pulled
# in via add_subdirectory() rather than being a standalone build).
--- ceph-19.2.0/src/googletest/googlemock/CMakeLists.txt.orig 2025-01-23 07:40:52.399853240 -0500
+++ ceph-19.2.0/src/googletest/googlemock/CMakeLists.txt 2025-01-23 07:43:18.466236642 -0500
@@ -54,6 +54,7 @@
# if they are the same (the default).
add_subdirectory("${gtest_dir}" "${gmock_BINARY_DIR}/${gtest_dir}")
+option(BUILD_SHARED_LIBS "Build shared libraries (DLLs)." OFF)
# These commands only run if this is the main project
if(CMAKE_PROJECT_NAME STREQUAL "gmock" OR CMAKE_PROJECT_NAME STREQUAL "googletest-distribution")
@@ -100,7 +101,7 @@
else()
cxx_library(gmock "${cxx_strict}" src/gmock-all.cc)
target_link_libraries(gmock PUBLIC gtest)
- set_target_properties(gmock PROPERTIES VERSION ${GOOGLETEST_VERSION})
+ # set_target_properties(gmock PROPERTIES VERSION ${GOOGLETEST_VERSION})
cxx_library(gmock_main "${cxx_strict}" src/gmock_main.cc)
target_link_libraries(gmock_main PUBLIC gmock)
set_target_properties(gmock_main PROPERTIES VERSION ${GOOGLETEST_VERSION})
-11
View File
@@ -1,11 +0,0 @@
--- ceph-19.2.0/src/tracing/CMakeLists.txt.orig 2025-01-23 17:51:43.874162594 -0500
+++ ceph-19.2.0/src/tracing/CMakeLists.txt 2025-01-23 17:54:21.877186735 -0500
@@ -6,6 +6,8 @@
set(header_dir ${working_dir}/tracing)
file(MAKE_DIRECTORY ${header_dir})
+set(CMAKE_C99_EXTENSION_COMPILE_OPTION "-std=gnu23")
+
add_custom_target(tracepoint_libraries)
file(GLOB tps "*.tp")
@@ -1,30 +0,0 @@
--- ceph-19.2.0/src/test/neorados/common_tests.h.orig 2024-09-18 12:27:51.000000000 -0400
+++ ceph-19.2.0/src/test/neorados/common_tests.h 2025-01-26 19:31:32.545762972 -0500
@@ -47,6 +47,18 @@
#include "gtest/gtest.h"
+#if !defined(GTEST_DISALLOW_COPY_AND_ASSIGN_)
+#define GTEST_DISALLOW_COPY_AND_ASSIGN_(type) \
+ type(type const&) = delete; \
+ type& operator=(type const&) = delete
+#endif
+
+#if !defined(GTEST_DISALLOW_MOVE_AND_ASSIGN_)
+#define GTEST_DISALLOW_MOVE_AND_ASSIGN_(type) \
+ type(type&&) noexcept = delete; \
+ type& operator=(type&&) noexcept = delete
+#endif
+
/// \file test/neorados/common_tests.h
///
/// \brief Tools for testing neorados code
@@ -398,7 +410,7 @@
\
private: \
boost::asio::awaitable<void> CoTestBody() override; \
- static ::testing::TestInfo *const test_info_ GTEST_ATTRIBUTE_UNUSED_; \
+ static ::testing::TestInfo *const test_info_ __attribute__((unused)); \
}; \
\
::testing::TestInfo *const GTEST_TEST_CLASS_NAME_(test_suite_name, \
-52
View File
@@ -1,52 +0,0 @@
--- ceph-19.2.2/src/pybind/cephfs/setup.py.orig 2025-05-29 15:51:00.656323008 -0400
+++ ceph-19.2.2/src/pybind/cephfs/setup.py 2025-05-29 15:52:13.215102123 -0400
@@ -1,11 +1,11 @@
import os
-import pkgutil
+import importlib.util
import shutil
import subprocess
import sys
import tempfile
import textwrap
-if not pkgutil.find_loader('setuptools'):
+if not importlib.util.find_spec('setuptools'):
from distutils.core import setup
from distutils.extension import Extension
else:
--- ceph-19.2.2/src/pybind/rbd/setup.py.orig 2025-05-29 15:48:35.863759314 -0400
+++ ceph-19.2.2/src/pybind/rbd/setup.py 2025-05-29 15:50:05.529742462 -0400
@@ -1,11 +1,11 @@
import os
-import pkgutil
+import importlib.util
import shutil
import subprocess
import sys
import tempfile
import textwrap
-if not pkgutil.find_loader('setuptools'):
+if not importlib.util.find_spec('setuptools'):
from distutils.core import setup
from distutils.extension import Extension
else:
--- ceph-19.2.2/src/pybind/rgw/setup.py.orig 2025-05-29 15:44:13.673781032 -0400
+++ ceph-19.2.2/src/pybind/rgw/setup.py 2025-05-29 15:48:23.444285313 -0400
@@ -1,5 +1,5 @@
-import pkgutil
-if not pkgutil.find_loader('setuptools'):
+import importlib.util
+if not importlib.util.find_spec('setuptools'):
from distutils.core import setup
from distutils.extension import Extension
else:
--- ceph-19.2.2/src/pybind/rados/setup.py.orig 2025-05-29 15:50:14.768095133 -0400
+++ ceph-19.2.2/src/pybind/rados/setup.py 2025-05-29 15:50:46.003687955 -0400
@@ -1,5 +1,5 @@
-import pkgutil
-if not pkgutil.find_loader('setuptools'):
+import importlib.util
+if not importlib.util.find_spec('setuptools'):
from distutils.core import setup
from distutils.extension import Extension
else:
-845
View File
@@ -1,845 +0,0 @@
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal_19.h ceph-19.2.2/src/s3select/include/encryption_internal_19.h
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal_19.h 2025-07-07 13:17:09.262618668 -0400
+++ ceph-19.2.2/src/s3select/include/encryption_internal_19.h 2025-07-07 13:08:00.746355226 -0400
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "parquet/properties.h"
+#include "parquet/types.h"
+
+using parquet::ParquetCipher;
+
+namespace parquet {
+namespace encryption {
+
+constexpr int kGcmTagLength = 16;
+constexpr int kNonceLength = 12;
+
+// Module types
+constexpr int8_t kFooter = 0;
+constexpr int8_t kColumnMetaData = 1;
+constexpr int8_t kDataPage = 2;
+constexpr int8_t kDictionaryPage = 3;
+constexpr int8_t kDataPageHeader = 4;
+constexpr int8_t kDictionaryPageHeader = 5;
+constexpr int8_t kColumnIndex = 6;
+constexpr int8_t kOffsetIndex = 7;
+
+/// Performs AES encryption operations with GCM or CTR ciphers.
+class AesEncryptor {
+ public:
+ static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
+ std::vector<AesEncryptor*>* all_encryptors);
+
+ ~AesEncryptor();
+
+ /// Size difference between plaintext and ciphertext, for this cipher.
+ int CiphertextSizeDelta();
+
+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
+ /// If different from value in constructor, exception will be thrown.
+ int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
+ int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
+
+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
+ int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
+ int key_len, const uint8_t* aad, int aad_len,
+ const uint8_t* nonce, uint8_t* encrypted_footer);
+
+ void WipeOut();
+
+ private:
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
+ explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
+ // PIMPL Idiom
+ class AesEncryptorImpl;
+ std::unique_ptr<AesEncryptorImpl> impl_;
+};
+
+/// Performs AES decryption operations with GCM or CTR ciphers.
+class AesDecryptor {
+ public:
+ static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
+ std::vector<AesDecryptor*>* all_decryptors);
+
+ ~AesDecryptor();
+ void WipeOut();
+
+ /// Size difference between plaintext and ciphertext, for this cipher.
+ int CiphertextSizeDelta();
+
+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
+ /// validation. If different from value in constructor, exception will be thrown.
+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
+ int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
+
+ private:
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
+ explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
+ // PIMPL Idiom
+ class AesDecryptorImpl;
+ std::unique_ptr<AesDecryptorImpl> impl_;
+};
+
+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
+ int16_t row_group_ordinal, int16_t column_ordinal,
+ int16_t page_ordinal);
+
+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
+
+// Update last two bytes of page (or page header) module AAD
+void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
+
+// Wraps OpenSSL RAND_bytes function
+void RandBytes(unsigned char* buf, int num);
+
+} // namespace encryption
+} // namespace parquet
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal_20.h ceph-19.2.2/src/s3select/include/encryption_internal_20.h
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal_20.h 2025-07-07 13:17:09.262618668 -0400
+++ ceph-19.2.2/src/s3select/include/encryption_internal_20.h 2025-07-07 13:08:00.746536155 -0400
@@ -0,0 +1,141 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "arrow/util/span.h"
+#include "parquet/properties.h"
+#include "parquet/types.h"
+
+using parquet::ParquetCipher;
+
+namespace parquet::encryption {
+
+constexpr int32_t kGcmTagLength = 16;
+constexpr int32_t kNonceLength = 12;
+
+// Module types
+constexpr int8_t kFooter = 0;
+constexpr int8_t kColumnMetaData = 1;
+constexpr int8_t kDataPage = 2;
+constexpr int8_t kDictionaryPage = 3;
+constexpr int8_t kDataPageHeader = 4;
+constexpr int8_t kDictionaryPageHeader = 5;
+constexpr int8_t kColumnIndex = 6;
+constexpr int8_t kOffsetIndex = 7;
+constexpr int8_t kBloomFilterHeader = 8;
+constexpr int8_t kBloomFilterBitset = 9;
+
+/// Performs AES encryption operations with GCM or CTR ciphers.
+class PARQUET_EXPORT AesEncryptor {
+ public:
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
+ /// If write_length is true, prepend ciphertext length to the ciphertext
+ explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
+ bool write_length = true);
+
+ static std::unique_ptr<AesEncryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
+ bool metadata, bool write_length = true);
+
+ ~AesEncryptor();
+
+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
+ [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const;
+
+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
+ /// If different from value in constructor, exception will be thrown.
+ int32_t Encrypt(::arrow::util::span<const uint8_t> plaintext,
+ ::arrow::util::span<const uint8_t> key,
+ ::arrow::util::span<const uint8_t> aad,
+ ::arrow::util::span<uint8_t> ciphertext);
+
+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
+ int32_t SignedFooterEncrypt(::arrow::util::span<const uint8_t> footer,
+ ::arrow::util::span<const uint8_t> key,
+ ::arrow::util::span<const uint8_t> aad,
+ ::arrow::util::span<const uint8_t> nonce,
+ ::arrow::util::span<uint8_t> encrypted_footer);
+
+ private:
+ // PIMPL Idiom
+ class AesEncryptorImpl;
+ std::unique_ptr<AesEncryptorImpl> impl_;
+};
+
+/// Performs AES decryption operations with GCM or CTR ciphers.
+class PARQUET_EXPORT AesDecryptor {
+ public:
+ /// \brief Construct an AesDecryptor
+ ///
+ /// \param alg_id the encryption algorithm to use
+ /// \param key_len key length. Possible values: 16, 24, 32 bytes.
+ /// \param metadata if true then this is a metadata decryptor
+ /// \param contains_length if true, expect ciphertext length prepended to the ciphertext
+ explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
+ bool contains_length = true);
+
+ static std::unique_ptr<AesDecryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
+ bool metadata);
+
+ ~AesDecryptor();
+
+ /// The size of the plaintext, for this cipher and the specified ciphertext length.
+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
+
+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
+
+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
+ /// validation. If different from value in constructor, exception will be thrown.
+ /// The caller is responsible for ensuring that the plaintext buffer is at least as
+ /// large as PlaintextLength(ciphertext_len).
+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
+ ::arrow::util::span<const uint8_t> key,
+ ::arrow::util::span<const uint8_t> aad,
+ ::arrow::util::span<uint8_t> plaintext);
+
+ private:
+ // PIMPL Idiom
+ class AesDecryptorImpl;
+ std::unique_ptr<AesDecryptorImpl> impl_;
+};
+
+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
+ int16_t row_group_ordinal, int16_t column_ordinal,
+ int32_t page_ordinal);
+
+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
+
+// Update last two bytes of page (or page header) module AAD
+void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD);
+
+// Wraps OpenSSL RAND_bytes function
+void RandBytes(unsigned char* buf, size_t num);
+
+// Ensure OpenSSL is initialized.
+//
+// This is only necessary in specific situations since OpenSSL otherwise
+// initializes itself automatically. For example, under Valgrind, a memory
+// leak will be reported if OpenSSL is initialized for the first time from
+// a worker thread; calling this function from the main thread prevents this.
+void EnsureBackendInitialized();
+
+} // namespace parquet::encryption
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal.h ceph-19.2.2/src/s3select/include/encryption_internal.h
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/encryption_internal.h 2025-07-07 13:08:00.745633149 -0400
@@ -17,98 +17,8 @@
#pragma once
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/properties.h"
-#include "parquet/types.h"
-
-using parquet::ParquetCipher;
-
-namespace parquet {
-namespace encryption {
-
-constexpr int kGcmTagLength = 16;
-constexpr int kNonceLength = 12;
-
-// Module types
-constexpr int8_t kFooter = 0;
-constexpr int8_t kColumnMetaData = 1;
-constexpr int8_t kDataPage = 2;
-constexpr int8_t kDictionaryPage = 3;
-constexpr int8_t kDataPageHeader = 4;
-constexpr int8_t kDictionaryPageHeader = 5;
-constexpr int8_t kColumnIndex = 6;
-constexpr int8_t kOffsetIndex = 7;
-
-/// Performs AES encryption operations with GCM or CTR ciphers.
-class AesEncryptor {
- public:
- static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
- std::vector<AesEncryptor*>* all_encryptors);
-
- ~AesEncryptor();
-
- /// Size difference between plaintext and ciphertext, for this cipher.
- int CiphertextSizeDelta();
-
- /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
- /// If different from value in constructor, exception will be thrown.
- int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
- int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
-
- /// Encrypts plaintext footer, in order to compute footer signature (tag).
- int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
- int key_len, const uint8_t* aad, int aad_len,
- const uint8_t* nonce, uint8_t* encrypted_footer);
-
- void WipeOut();
-
- private:
- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
- explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
- // PIMPL Idiom
- class AesEncryptorImpl;
- std::unique_ptr<AesEncryptorImpl> impl_;
-};
-
-/// Performs AES decryption operations with GCM or CTR ciphers.
-class AesDecryptor {
- public:
- static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
- std::vector<AesDecryptor*>* all_decryptors);
-
- ~AesDecryptor();
- void WipeOut();
-
- /// Size difference between plaintext and ciphertext, for this cipher.
- int CiphertextSizeDelta();
-
- /// Decrypts ciphertext with the key and aad. Key length is passed only for
- /// validation. If different from value in constructor, exception will be thrown.
- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
- int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
-
- private:
- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
- explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
- // PIMPL Idiom
- class AesDecryptorImpl;
- std::unique_ptr<AesDecryptorImpl> impl_;
-};
-
-std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
- int16_t row_group_ordinal, int16_t column_ordinal,
- int16_t page_ordinal);
-
-std::string CreateFooterAad(const std::string& aad_prefix_bytes);
-
-// Update last two bytes of page (or page header) module AAD
-void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
-
-// Wraps OpenSSL RAND_bytes function
-void RandBytes(unsigned char* buf, int num);
-
-} // namespace encryption
-} // namespace parquet
+#if ARROW_VERSION_MAJOR < 20
+#include "encryption_internal_19.h"
+#else
+#include "encryption_internal_20.h"
+#endif
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_19.h ceph-19.2.2/src/s3select/include/internal_file_decryptor_19.h
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_19.h 2025-07-07 13:17:58.730837390 -0400
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor_19.h 2025-07-07 13:08:00.746893764 -0400
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "parquet/schema.h"
+
+namespace parquet {
+
+namespace encryption {
+class AesDecryptor;
+class AesEncryptor;
+} // namespace encryption
+
+class FileDecryptionProperties;
+
+class PARQUET_EXPORT Decryptor {
+ public:
+ Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
+ const std::string& file_aad, const std::string& aad,
+ ::arrow::MemoryPool* pool);
+
+ const std::string& file_aad() const { return file_aad_; }
+ void UpdateAad(const std::string& aad) { aad_ = aad; }
+ ::arrow::MemoryPool* pool() { return pool_; }
+
+ int CiphertextSizeDelta();
+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
+
+ private:
+ encryption::AesDecryptor* aes_decryptor_;
+ std::string key_;
+ std::string file_aad_;
+ std::string aad_;
+ ::arrow::MemoryPool* pool_;
+};
+
+class InternalFileDecryptor {
+ public:
+ explicit InternalFileDecryptor(FileDecryptionProperties* properties,
+ const std::string& file_aad,
+ ParquetCipher::type algorithm,
+ const std::string& footer_key_metadata,
+ ::arrow::MemoryPool* pool);
+
+ std::string& file_aad() { return file_aad_; }
+
+ std::string GetFooterKey();
+
+ ParquetCipher::type algorithm() { return algorithm_; }
+
+ std::string& footer_key_metadata() { return footer_key_metadata_; }
+
+ FileDecryptionProperties* properties() { return properties_; }
+
+ void WipeOutDecryptionKeys();
+
+ ::arrow::MemoryPool* pool() { return pool_; }
+
+ std::shared_ptr<Decryptor> GetFooterDecryptor();
+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
+ std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
+ const std::string& column_path, const std::string& column_key_metadata,
+ const std::string& aad = "");
+ std::shared_ptr<Decryptor> GetColumnDataDecryptor(
+ const std::string& column_path, const std::string& column_key_metadata,
+ const std::string& aad = "");
+
+ private:
+ FileDecryptionProperties* properties_;
+ // Concatenation of aad_prefix (if exists) and aad_file_unique
+ std::string file_aad_;
+ std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
+ std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
+
+ std::shared_ptr<Decryptor> footer_metadata_decryptor_;
+ std::shared_ptr<Decryptor> footer_data_decryptor_;
+ ParquetCipher::type algorithm_;
+ std::string footer_key_metadata_;
+ std::vector<encryption::AesDecryptor*> all_decryptors_;
+
+ /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
+ // types of meta_decryptors and data_decryptors.
+ std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
+ std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
+
+ ::arrow::MemoryPool* pool_;
+
+ std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
+ std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
+ const std::string& column_key_metadata,
+ const std::string& aad,
+ bool metadata = false);
+
+ encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
+ encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
+
+ int MapKeyLenToDecryptorArrayIndex(int key_len);
+};
+
+} // namespace parquet
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_20.h ceph-19.2.2/src/s3select/include/internal_file_decryptor_20.h
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_20.h 2025-07-07 13:17:58.730837390 -0400
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor_20.h 2025-07-07 13:08:00.747082281 -0400
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "parquet/schema.h"
+
+namespace parquet {
+
+namespace encryption {
+class AesDecryptor;
+class AesEncryptor;
+} // namespace encryption
+
+class ColumnCryptoMetaData;
+class FileDecryptionProperties;
+
+// An object handling decryption using well-known encryption parameters
+//
+// CAUTION: Decryptor objects are not thread-safe.
+class PARQUET_EXPORT Decryptor {
+ public:
+ Decryptor(std::unique_ptr<encryption::AesDecryptor> decryptor, const std::string& key,
+ const std::string& file_aad, const std::string& aad,
+ ::arrow::MemoryPool* pool);
+ ~Decryptor();
+
+ const std::string& file_aad() const { return file_aad_; }
+ void UpdateAad(const std::string& aad) { aad_ = aad; }
+ ::arrow::MemoryPool* pool() { return pool_; }
+
+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
+ ::arrow::util::span<uint8_t> plaintext);
+
+ private:
+ std::unique_ptr<encryption::AesDecryptor> aes_decryptor_;
+ std::string key_;
+ std::string file_aad_;
+ std::string aad_;
+ ::arrow::MemoryPool* pool_;
+};
+
+class InternalFileDecryptor {
+ public:
+ explicit InternalFileDecryptor(std::shared_ptr<FileDecryptionProperties> properties,
+ const std::string& file_aad,
+ ParquetCipher::type algorithm,
+ const std::string& footer_key_metadata,
+ ::arrow::MemoryPool* pool);
+
+ const std::string& file_aad() const { return file_aad_; }
+
+ std::string GetFooterKey();
+
+ ParquetCipher::type algorithm() const { return algorithm_; }
+
+ const std::string& footer_key_metadata() const { return footer_key_metadata_; }
+
+ const std::shared_ptr<FileDecryptionProperties>& properties() const {
+ return properties_;
+ }
+
+ ::arrow::MemoryPool* pool() const { return pool_; }
+
+ // Get a Decryptor instance for the Parquet footer
+ std::unique_ptr<Decryptor> GetFooterDecryptor();
+
+ // Get a Decryptor instance for column chunk metadata.
+ std::unique_ptr<Decryptor> GetColumnMetaDecryptor(
+ const std::string& column_path, const std::string& column_key_metadata,
+ const std::string& aad = "") {
+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/true);
+ }
+
+ // Get a Decryptor instance for column chunk data.
+ std::unique_ptr<Decryptor> GetColumnDataDecryptor(
+ const std::string& column_path, const std::string& column_key_metadata,
+ const std::string& aad = "") {
+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/false);
+ }
+
+ // Get a Decryptor factory for column chunk metadata.
+ //
+ // This is typically useful if multi-threaded decryption is expected.
+ // This is a static function as it accepts a null `InternalFileDecryptor*`
+ // argument if the column is not encrypted.
+ static std::function<std::unique_ptr<Decryptor>()> GetColumnMetaDecryptorFactory(
+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
+ const std::string& aad = "");
+ // Get a Decryptor factory for column chunk data.
+ //
+ // This is typically useful if multi-threaded decryption is expected.
+ // This is a static function as it accepts a null `InternalFileDecryptor*`
+ // argument if the column is not encrypted.
+ static std::function<std::unique_ptr<Decryptor>()> GetColumnDataDecryptorFactory(
+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
+ const std::string& aad = "");
+
+ private:
+ std::shared_ptr<FileDecryptionProperties> properties_;
+ // Concatenation of aad_prefix (if exists) and aad_file_unique
+ std::string file_aad_;
+ ParquetCipher::type algorithm_;
+ std::string footer_key_metadata_;
+ ::arrow::MemoryPool* pool_;
+
+ // Protects footer_key_ updates
+ std::mutex mutex_;
+ std::string footer_key_;
+
+ std::string GetColumnKey(const std::string& column_path,
+ const std::string& column_key_metadata);
+
+ std::unique_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
+
+ std::unique_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
+ const std::string& column_key_metadata,
+ const std::string& aad, bool metadata);
+
+ std::function<std::unique_ptr<Decryptor>()> GetColumnDecryptorFactory(
+ const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata);
+};
+
+void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal,
+ int16_t column_ordinal, int8_t module_type);
+
+} // namespace parquet
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h ceph-19.2.2/src/s3select/include/internal_file_decryptor.h
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor.h 2025-07-07 13:08:00.746720961 -0400
@@ -17,105 +17,8 @@
#pragma once
-#include <map>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "parquet/schema.h"
-
-namespace parquet {
-
-namespace encryption {
-class AesDecryptor;
-class AesEncryptor;
-} // namespace encryption
-
-class FileDecryptionProperties;
-
-class PARQUET_EXPORT Decryptor {
- public:
- Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
- const std::string& file_aad, const std::string& aad,
- ::arrow::MemoryPool* pool);
-
- const std::string& file_aad() const { return file_aad_; }
- void UpdateAad(const std::string& aad) { aad_ = aad; }
- ::arrow::MemoryPool* pool() { return pool_; }
-
- int CiphertextSizeDelta();
- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
-
- private:
- encryption::AesDecryptor* aes_decryptor_;
- std::string key_;
- std::string file_aad_;
- std::string aad_;
- ::arrow::MemoryPool* pool_;
-};
-
-class InternalFileDecryptor {
- public:
- explicit InternalFileDecryptor(FileDecryptionProperties* properties,
- const std::string& file_aad,
- ParquetCipher::type algorithm,
- const std::string& footer_key_metadata,
- ::arrow::MemoryPool* pool);
-
- std::string& file_aad() { return file_aad_; }
-
- std::string GetFooterKey();
-
- ParquetCipher::type algorithm() { return algorithm_; }
-
- std::string& footer_key_metadata() { return footer_key_metadata_; }
-
- FileDecryptionProperties* properties() { return properties_; }
-
- void WipeOutDecryptionKeys();
-
- ::arrow::MemoryPool* pool() { return pool_; }
-
- std::shared_ptr<Decryptor> GetFooterDecryptor();
- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
- std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
- const std::string& column_path, const std::string& column_key_metadata,
- const std::string& aad = "");
- std::shared_ptr<Decryptor> GetColumnDataDecryptor(
- const std::string& column_path, const std::string& column_key_metadata,
- const std::string& aad = "");
-
- private:
- FileDecryptionProperties* properties_;
- // Concatenation of aad_prefix (if exists) and aad_file_unique
- std::string file_aad_;
- std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
- std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
-
- std::shared_ptr<Decryptor> footer_metadata_decryptor_;
- std::shared_ptr<Decryptor> footer_data_decryptor_;
- ParquetCipher::type algorithm_;
- std::string footer_key_metadata_;
- std::vector<encryption::AesDecryptor*> all_decryptors_;
-
- /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
- // types of meta_decryptors and data_decryptors.
- std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
- std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
-
- ::arrow::MemoryPool* pool_;
-
- std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
- std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
- const std::string& column_key_metadata,
- const std::string& aad,
- bool metadata = false);
-
- encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
- encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
-
- int MapKeyLenToDecryptorArrayIndex(int key_len);
-};
-
-} // namespace parquet
+#if ARROW_VERSION_MAJOR < 20
+#include "internal_file_decryptor_19.h"
+#else
+#include "internal_file_decryptor_20.h"
+#endif
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h
--- ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h 2024-03-26 19:23:12.000000000 -0400
+++ ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h 2025-07-07 13:08:00.747275733 -0400
@@ -983,6 +1002,7 @@
throw ParquetException("Encrypted files cannot contain more than 32767 row groups");
}
+#if ARROW_VERSION_MAJOR < 20
// The column is encrypted
std::shared_ptr<::parquet::Decryptor> meta_decryptor;
std::shared_ptr<Decryptor> data_decryptor;
@@ -1016,6 +1036,25 @@
false,
#endif
properties_.memory_pool(), &ctx);
+#else
+ // Arrow 20+ version uses factory functions instead of shared_ptr for decryptors
+ std::function<std::unique_ptr<Decryptor>()> meta_decryptor_factory =
+ InternalFileDecryptor::GetColumnMetaDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
+ std::function<std::unique_ptr<Decryptor>()> data_decryptor_factory =
+ InternalFileDecryptor::GetColumnDataDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
+
+ const CryptoContext ctx {
+ col->has_dictionary_page(),
+ row_group_ordinal_,
+ static_cast<int16_t>(i),
+ meta_decryptor_factory,
+ data_decryptor_factory,
+ };
+
+ return PageReader::Open(stream, col->num_values(), col->compression(),
+ false,
+ properties_.memory_pool(), &ctx);
+#endif
}
private:
@@ -1052,7 +1091,9 @@
}
void Close() override {
+#if ARROW_VERSION_MAJOR < 20
if (file_decryptor_) file_decryptor_->WipeOutDecryptionKeys();
+#endif
}
std::shared_ptr<RowGroupReader> GetRowGroup(int i) override {
@@ -1230,9 +1271,17 @@
// Handle AAD prefix
EncryptionAlgorithm algo = file_crypto_metadata->encryption_algorithm();
std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
+#if ARROW_VERSION_MAJOR < 20
file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
file_decryption_properties, file_aad, algo.algorithm,
file_crypto_metadata->key_metadata(), properties_.memory_pool());
+#else
+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
+ file_aad, algo.algorithm,
+ file_crypto_metadata->key_metadata(), properties_.memory_pool());
+#endif
int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len;
uint32_t metadata_len = footer_len - crypto_metadata_len;
@@ -1259,9 +1312,18 @@
EncryptionAlgorithm algo = file_metadata_->encryption_algorithm();
// Handle AAD prefix
std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
+#if ARROW_VERSION_MAJOR < 20
file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
file_decryption_properties, file_aad, algo.algorithm,
file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
+#else
+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
+ file_aad, algo.algorithm,
+ file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
+ // In Arrow 20+, no need to set file_decryptor in metadata
+#endif
// set the InternalFileDecryptor in the metadata as well, as it's used
// for signature verification and for ColumnChunkMetaData creation.
#if GAL_set_file_decryptor_declare_private
@@ -1,37 +0,0 @@
From 8c78a22d2cf69892570f635735d9735169b64a75 Mon Sep 17 00:00:00 2001
From: Peter Sabaini <peter.sabaini@canonical.com>
Date: Wed, 11 Sep 2024 16:56:50 +0200
Subject: [PATCH] ceph-volume: fix importlib.metadata compat
The importlib.metadata library removed older shims in releases >5.0.0
where EntryPoints objects use .select() instead of dict-like access.
Fixes: https://tracker.ceph.com/issues/68032
Signed-off-by: Peter Sabaini <peter.sabaini@canonical.com>
---
src/ceph-volume/ceph_volume/main.py | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/ceph-volume/ceph_volume/main.py b/src/ceph-volume/ceph_volume/main.py
index f8eca65ec497c..4f27f429e89e2 100644
--- a/src/ceph-volume/ceph_volume/main.py
+++ b/src/ceph-volume/ceph_volume/main.py
@@ -11,8 +11,16 @@
from importlib.metadata import entry_points
def get_entry_points(group: str): # type: ignore
- return entry_points().get(group, []) # type: ignore
+ eps = entry_points()
+ if hasattr(eps, 'select'):
+ # New importlib.metadata uses .select()
+ return eps.select(group=group)
+ else:
+ # Fallback to older EntryPoints that returns dicts
+ return eps.get(group, []) # type: ignore
+
except ImportError:
+ # Fallback to `pkg_resources` for older versions
from pkg_resources import iter_entry_points as entry_points # type: ignore
def get_entry_points(group: str): # type: ignore
@@ -1,53 +0,0 @@
From 5924df771f850c249396a37a62f97ac242bf2f96 Mon Sep 17 00:00:00 2001
From: Hector Martin <marcan@marcan.st>
Date: Wed, 20 Aug 2025 13:34:08 +0900
Subject: [PATCH 1/2] mgr/dashboard: Make saml2 robust against module load
errors
Loading saml2 can fail due to issues with the xmlsec package (possibly
subinterpreter related):
File "/usr/share/ceph/mgr/dashboard/controllers/saml2.py", line 6, in <module>
from onelogin.saml2.auth import OneLogin_Saml2_Auth
File "/lib/python3.13/site-packages/onelogin/saml2/auth.py", line 12, in <module>
import xmlsec
xmlsec.Error: (100, 'lxml & xmlsec libxml2 library version mismatch')
Instead of taking down the entire dashboard module, treat this exception
like a missing saml2 package.
Signed-off-by: Hector Martin <marcan@marcan.st>
---
src/pybind/mgr/dashboard/controllers/saml2.py | 2 +-
src/pybind/mgr/dashboard/services/sso.py | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/pybind/mgr/dashboard/controllers/saml2.py b/src/pybind/mgr/dashboard/controllers/saml2.py
index c11b18a27bc7..e859f558cdc4 100644
--- a/src/pybind/mgr/dashboard/controllers/saml2.py
+++ b/src/pybind/mgr/dashboard/controllers/saml2.py
@@ -8,7 +8,7 @@ try:
from onelogin.saml2.settings import OneLogin_Saml2_Settings
python_saml_imported = True
-except ImportError:
+except Exception:
python_saml_imported = False
from .. import mgr
diff --git a/src/pybind/mgr/dashboard/services/sso.py b/src/pybind/mgr/dashboard/services/sso.py
index 2290e6ea3e15..38910ca4aa34 100644
--- a/src/pybind/mgr/dashboard/services/sso.py
+++ b/src/pybind/mgr/dashboard/services/sso.py
@@ -20,7 +20,7 @@ try:
from onelogin.saml2.settings import OneLogin_Saml2_Settings as Saml2Settings
python_saml_imported = True
-except ImportError:
+except Exception:
python_saml_imported = False
--
2.50.1
@@ -1,44 +0,0 @@
From 76af91b516bb31ae0a79f55f315bef2c2105a06e Mon Sep 17 00:00:00 2001
From: Nizamudeen A <nia@redhat.com>
Date: Thu, 26 Jun 2025 12:55:22 +0530
Subject: [PATCH 2/2] mgr/dashboard: catch protobuf error due to mismatch in
version
Signed-off-by: Nizamudeen A <nia@redhat.com>
(cherry picked from commit acb0f19c4a50b2ea68f328a61a14a2da06be298b)
Conflicts:
src/pybind/mgr/dashboard/services/nvmeof_client.py
- only kept relavant portion applicable for squid
---
src/pybind/mgr/dashboard/services/nvmeof_client.py | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/src/pybind/mgr/dashboard/services/nvmeof_client.py b/src/pybind/mgr/dashboard/services/nvmeof_client.py
index e0ea6d1e48b3..be2b5edbaa52 100644
--- a/src/pybind/mgr/dashboard/services/nvmeof_client.py
+++ b/src/pybind/mgr/dashboard/services/nvmeof_client.py
@@ -1,3 +1,5 @@
+# pylint: disable=unexpected-keyword-arg
+
import functools
import logging
from collections.abc import Iterable
@@ -9,6 +11,14 @@ from .nvmeof_conf import NvmeofGatewaysConfig
logger = logging.getLogger("nvmeof_client")
try:
+ # if the protobuf version is newer than what we generated with
+ # proto file import will fail (because of differences between what's
+ # available in centos and ubuntu).
+ # this "hack" should be removed once we update both the
+ # distros; centos and ubuntu.
+ import os
+ os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
+
import grpc # type: ignore
import grpc._channel # type: ignore
from google.protobuf.message import Message # type: ignore
--
2.50.1
@@ -1,152 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "Max R. Carrara" <m.carrara@proxmox.com>
Date: Wed, 16 Jul 2025 13:14:39 +0200
Subject: [PATCH 58/59] pybind/mgr/restful: provide workaround for PyO3
ImportError
Move the self-signed cert generation into a separate module
inside python-common/ceph and run the module in a separate Python
process.
This provides a workaround for the ImportError thrown by PyO3 when
the `restful` module is loaded in the context of multiple Python
sub-interpreters being present. In particular, the ImportError is
thrown by the `crypto` module of the `OpenSSL` package.
Inspired by an upstream PR [0].
[0]: https://github.com/ceph/ceph/pull/62951
Signed-off-by: Max R. Carrara <m.carrara@proxmox.com>
---
src/pybind/mgr/restful/module.py | 24 +++------
src/python-common/ceph/_crypto_wrap.py | 69 ++++++++++++++++++++++++++
2 files changed, 76 insertions(+), 17 deletions(-)
create mode 100644 src/python-common/ceph/_crypto_wrap.py
diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py
index 0f8c78e0bd8..7f93c41f1e6 100644
--- a/src/pybind/mgr/restful/module.py
+++ b/src/pybind/mgr/restful/module.py
@@ -7,6 +7,7 @@ import json
import time
import errno
import inspect
+import subprocess
import tempfile
import threading
import traceback
@@ -19,7 +20,6 @@ from . import context
from uuid import uuid4
from pecan import jsonify, make_app
-from OpenSSL import crypto
from pecan.rest import RestController
from werkzeug.serving import make_server, make_ssl_devcert
@@ -401,24 +401,14 @@ class Module(MgrModule):
def create_self_signed_cert(self):
- # create a key pair
- pkey = crypto.PKey()
- pkey.generate_key(crypto.TYPE_RSA, 2048)
-
- # create a self-signed cert
- cert = crypto.X509()
- cert.get_subject().O = "IT"
- cert.get_subject().CN = "ceph-restful"
- cert.set_serial_number(int(uuid4()))
- cert.gmtime_adj_notBefore(0)
- cert.gmtime_adj_notAfter(10*365*24*60*60)
- cert.set_issuer(cert.get_subject())
- cert.set_pubkey(pkey)
- cert.sign(pkey, 'sha512')
+ cmd = ["python3", "-m", "ceph._crypto_wrap", "create_self_signed_cert"]
+
+ response = subprocess.run(cmd, capture_output=True, check=True)
+ response_obj = json.loads(response.stdout)
return (
- crypto.dump_certificate(crypto.FILETYPE_PEM, cert),
- crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
+ response_obj["cert"].encode("utf-8"),
+ response_obj["key"].encode("utf-8"),
)
diff --git a/src/python-common/ceph/_crypto_wrap.py b/src/python-common/ceph/_crypto_wrap.py
new file mode 100644
index 00000000000..16a19a5345e
--- /dev/null
+++ b/src/python-common/ceph/_crypto_wrap.py
@@ -0,0 +1,69 @@
+"""CLI wrapper for cryptographic functions of the :mod:`restful` module.
+
+To be called via :func:`subprocess.run()` as a workaround for
+:class:`ImportError`s related to PyO3's current lack of sub-interpreter
+support.
+
+Note:
+ Since this module is installed as part of the ``ceph`` package,
+ it should be called like so::
+
+ python3 -m ceph._crypto_wrap create_self_signed_cert
+"""
+
+import argparse
+import sys
+import json
+
+from argparse import Namespace
+from typing import Any
+from uuid import uuid4
+
+from OpenSSL import crypto
+
+
+def _respond(data: dict[str, Any]) -> None:
+ json.dump(data, sys.stdout)
+ sys.stdout.flush()
+
+
+def create_self_signed_cert(args: Namespace) -> None:
+ cert_key_pair = _create_self_signed_cert()
+ _respond(cert_key_pair)
+
+
+def _create_self_signed_cert() -> dict[str, str]:
+ # create a key pair
+ pubkey = crypto.PKey()
+ pubkey.generate_key(crypto.TYPE_RSA, 2048)
+
+ # create a self-signed cert
+ cert = crypto.X509()
+ cert.get_subject().O = "IT"
+ cert.get_subject().CN = "ceph-restful"
+ cert.set_serial_number(int(uuid4()))
+ cert.gmtime_adj_notBefore(0)
+ cert.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60)
+ cert.set_issuer(cert.get_subject())
+ cert.set_pubkey(pubkey)
+ cert.sign(pubkey, "sha512")
+
+ return {
+ "cert": crypto.dump_certificate(crypto.FILETYPE_PEM, cert).decode(),
+ "key": crypto.dump_privatekey(crypto.FILETYPE_PEM, pubkey).decode(),
+ }
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(prog="_crypto_wrap.py")
+ subparsers = parser.add_subparsers(required=True)
+
+ parser_cssc = subparsers.add_parser("create_self_signed_cert")
+ parser_cssc.set_defaults(func=create_self_signed_cert)
+
+ args = parser.parse_args()
+ args.func(args)
+
+
+if __name__ == "__main__":
+ main()
@@ -1,56 +0,0 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: "Max R. Carrara" <m.carrara@proxmox.com>
Date: Wed, 16 Jul 2025 16:31:43 +0200
Subject: [PATCH 59/59] mgr: fix module import by making NOTIFY_TYPES in py
modules optional
If NOTIFY_TYPES isn't an attribute of the passed class, the Python
(sub-)interpreter raises an AttributeError that must be handled or cleared
explicitly via the Python C-API. Unfortunately, this isn't done here,
which means that the exception sticks around until handled.
This caused a call to PyModule::load_subclass_of() to fail and
incorrectly report the AttributeError as cause.
Checking whether the class has NOTIFY_TYPES as attribute in the first
place fixes this.
Note that there's an upstream PR [0] that wasn't backported that aimed
to fix this, but does so incorrectly, as the exception is still not
cleared there. The warnings regarding NOTIFY_TYPES missing also occurs
on Reef but doesn't cause any module imports to fail there. As the
affected Ceph code has stayed mostly the same between bookworm and
trixie releases, this suggests that some behavior between Python 3.11
and 3.13 likely changed.
Either way, avoiding the AttributeError altogether fixes this.
[0]: https://github.com/ceph/ceph/pull/57106
Signed-off-by: Max R. Carrara <m.carrara@proxmox.com>
---
src/mgr/PyModule.cc | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc
index 084cf3ffc1e..e6fd269dca5 100644
--- a/src/mgr/PyModule.cc
+++ b/src/mgr/PyModule.cc
@@ -513,11 +513,13 @@ int PyModule::register_options(PyObject *cls)
int PyModule::load_notify_types()
{
- PyObject *ls = PyObject_GetAttrString(pClass, "NOTIFY_TYPES");
- if (ls == nullptr) {
- derr << "Module " << get_name() << " has missing NOTIFY_TYPES member" << dendl;
- return -EINVAL;
+ if (!PyObject_HasAttrString(pClass, "NOTIFY_TYPES")) {
+ dout(10) << "Module " << get_name() << " has no NOTIFY_TYPES member" << dendl;
+ return 0;
}
+
+ PyObject *ls = PyObject_GetAttrString(pClass, "NOTIFY_TYPES");
+
if (!PyObject_TypeCheck(ls, &PyList_Type)) {
// Relatively easy mistake for human to make, e.g. defining COMMANDS
// as a {} instead of a []
+1175 -501
View File
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,13 @@
diff --git a/cmake/modules/BuildBoost.cmake b/cmake/modules/BuildBoost.cmake
index 2e92132366..3cb1e3d958 100644
--- a/cmake/modules/BuildBoost.cmake
+++ b/cmake/modules/BuildBoost.cmake
@@ -62,7 +62,7 @@ function(do_build_boost version)
else()
list(APPEND boost_features "address-model=32")
endif()
- set(BOOST_CXXFLAGS "-fPIC -w") # check on arm, etc <---XXX
+ set(BOOST_CXXFLAGS "-fPIC -w -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection -ftemplate-depth-1024 -fno-new-ttp-matching -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free") # check on arm, etc <---XXX
list(APPEND boost_features "cxxflags=${BOOST_CXXFLAGS}")
set(boost_with_libs)
+1 -1
View File
@@ -1 +1 @@
SHA512 (ceph-19.2.3.tar.gz) = 278101d2df7bed5363b20c2b065d7a7b26252c8164511257e213ffaa58d509015558183de10bc9281bcbe4d9f85244bcac5bba4db9823e28df6a96d0b687d00a
SHA512 (ceph-17.2.7.tar.gz) = 9005de7661c6a9d1054f1388cff2f48eb1af93565ebf852669de546eed594ed6a2668944b25a64c406d00f88bd2f7a6eac2ae96b04789c03ea4d248d1a683e7b