fedor/Notes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Vendor dependencies

Browse Source
This commit is contained in:
fedor
2026-08-01 16:11:49 +03:00
parent 7f139a0241
commit 6b5e7f0f8b
29706 changed files with 9575646 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notes-service/vendor/arrow-schema/.cargo-checksum.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"$comment":"This file only protects against accidental modifications. It is not a security mechanism and does not protect against malicious changes.","files":{"Cargo.lock":"f35948ddbbb8eb403f30488e7f14fcc9052905d07a83b69d586da4ff9fd934ce","Cargo.toml":"f3f2dcdd3fbfb37f2c9b32c6b100a42d4257250797401eb77ac1f01d42a01c63","Cargo.toml.orig":"e089ef7ad631acdd4b305efb6912d5dec94a54409b0a5133ef412cc8839ee372","LICENSE.txt":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","NOTICE.txt":"f5b6a58f5d2082e403c49663d87566328d40cf03f16abb01f52632a09b97c55e","benches/ffi.rs":"14432b5a9f7d29dd85deabca58021244523e916ea456cf993b347a4f4a9c8db3","src/datatype.rs":"8f8f22dca6e3a0f0f80f74d877b9ace5bf33bf5e29dba8bce6ee8a06014f47a1","src/datatype_display.rs":"012b1aceecf43a919d15023fe162e21ba38636a5bbfd2d7668da25cdf6cf8ce0","src/datatype_parse.rs":"f8afd183edf6284cfb5fc4777f5da8db7f1199b276f68896c1e043971ba6b0b7","src/error.rs":"ea0a3f25853ef5cbcbea9cc1f5da49f1fc46a4fd463e4f954c42189d3e9f172c","src/extension/canonical/bool8.rs":"734019f639dfc444bc014bbc8427a216e1e0ae332ba9a1691c7e82401e410f4f","src/extension/canonical/fixed_shape_tensor.rs":"61debb239dbc6affec7e8f96e7d79d3bdf0dca1958c744f170f9a56186764ef9","src/extension/canonical/json.rs":"9732d0eedad47d9939b5ad8a2841fb4c863e2d7afb03bf103def18b6cd960b71","src/extension/canonical/mod.rs":"3eed6a72eebdc5712449da6007c97be1e9013f85e9eadb7f54550c7080af59e9","src/extension/canonical/opaque.rs":"e0c1b4f69e76544d5e6ac9b08ee31fc5a5fb93a708a84ba721b6f3357a2d9f7b","src/extension/canonical/uuid.rs":"88fd40076955de1158367750e193c29fa094984d312bd5697044ee1bc50f5db6","src/extension/canonical/variable_shape_tensor.rs":"be483eeb689f9d2ea652334c454f9740f50949259de12161c712d829e42aa9f8","src/extension/mod.rs":"9988cc4582ca8aee3c4a6f81a7d5e58b8210016458dec6c86a48d66747560700","src/ffi.rs":"f2e37165bd4bf18d7c8358a89bd8107a5c29a3009335be2cd96ed7b0cfa28c12","src/field.rs":"0f6d65d9a290eb388fb9826aabe1745ed107ca099b88a8c9fcd21275f2d5342a","src/fields.rs":"4882abd09049fc33c09f2886b831ce66270e49f8ec0bd2daa4708fad6051cb3a","src/lib.rs":"fcf571d02309a9f0f1a60b3e54b9271619835e5525c4247d7b3743cdbee62942","src/schema.rs":"0ce39e9a147c98fff1adc1c93517904592a3d8e33dc542164227d7596dcd13e9"},"package":"e4cf0d4a6609679e03002167a61074a21d7b1ad9ea65e462b2c0a97f8a3b2bc6"}
notes-service/vendor/arrow-schema/Cargo.lock
Generated Vendored
+821
View File
@@ -0,0 +1,821 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
dependencies = [
"memchr",
]
[[package]]
name = "alloca"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
dependencies = [
"cc",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anyhow"
version = "1.0.102"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c"
[[package]]
name = "arrow-schema"
version = "57.3.1"
dependencies = [
"bincode",
"bitflags",
"criterion",
"insta",
"serde",
"serde_core",
"serde_json",
]
[[package]]
name = "autocfg"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"serde",
"unty",
]
[[package]]
name = "bitflags"
version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.2.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
dependencies = [
"find-msvc-tools",
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "console"
version = "0.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
dependencies = [
"encode_unicode",
"libc",
"windows-sys",
]
[[package]]
name = "criterion"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
dependencies = [
"alloca",
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"itertools",
"num-traits",
"oorandom",
"page_size",
"regex",
"serde",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "encode_unicode"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "find-msvc-tools"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "getrandom"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasip2",
"wasip3",
]
[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b"
dependencies = [
"cfg-if",
"crunchy",
"zerocopy",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"foldhash",
]
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "heck"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "id-arena"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
[[package]]
name = "indexmap"
version = "2.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
dependencies = [
"equivalent",
"hashbrown 0.17.1",
"serde",
"serde_core",
]
[[package]]
name = "insta"
version = "1.47.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b4a6248eb93a4401ed2f37dfe8ea592d3cf05b7cf4f8efa867b6895af7e094e"
dependencies = [
"console",
"once_cell",
"similar",
"tempfile",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "leb128fmt"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "libc"
version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "log"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "page_size"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "prettyplease"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
"syn",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
name = "rustix"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "semver"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
"serde_derive",
]
[[package]]
name = "serde_core"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.149"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "similar"
version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "unicode-ident"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasip2"
version = "1.0.3+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
dependencies = [
"wit-bindgen 0.57.1",
]
[[package]]
name = "wasip3"
version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
"wit-bindgen 0.51.0",
]
[[package]]
name = "wasm-encoder"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319"
dependencies = [
"leb128fmt",
"wasmparser",
]
[[package]]
name = "wasm-metadata"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
"indexmap",
"wasm-encoder",
"wasmparser",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags",
"hashbrown 0.15.5",
"indexmap",
"semver",
]
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
name = "windows-sys"
version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
dependencies = [
"windows-link",
]
[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
dependencies = [
"wit-bindgen-rust-macro",
]
[[package]]
name = "wit-bindgen"
version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc"
dependencies = [
"anyhow",
"heck",
"wit-parser",
]
[[package]]
name = "wit-bindgen-rust"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
"indexmap",
"prettyplease",
"syn",
"wasm-metadata",
"wit-bindgen-core",
"wit-component",
]
[[package]]
name = "wit-bindgen-rust-macro"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a"
dependencies = [
"anyhow",
"prettyplease",
"proc-macro2",
"quote",
"syn",
"wit-bindgen-core",
"wit-bindgen-rust",
]
[[package]]
name = "wit-component"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags",
"indexmap",
"log",
"serde",
"serde_derive",
"serde_json",
"wasm-encoder",
"wasm-metadata",
"wasmparser",
"wit-parser",
]
[[package]]
name = "wit-parser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
"indexmap",
"log",
"semver",
"serde",
"serde_derive",
"serde_json",
"unicode-xid",
"wasmparser",
]
[[package]]
name = "zerocopy"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
notes-service/vendor/arrow-schema/Cargo.toml
Vendored
+101
View File
@@ -0,0 +1,101 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2024"
rust-version = "1.85"
name = "arrow-schema"
version = "57.3.1"
authors = ["Apache Arrow <dev@arrow.apache.org>"]
build = false
include = [
"benches/*.rs",
"src/**/*.rs",
"Cargo.toml",
"LICENSE.txt",
"NOTICE.txt",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "Defines the logical types for arrow arrays"
homepage = "https://github.com/apache/arrow-rs"
readme = false
keywords = ["arrow"]
license = "Apache-2.0"
repository = "https://github.com/apache/arrow-rs"
resolver = "2"
[package.metadata.docs.rs]
all-features = true
[features]
canonical_extension_types = [
"dep:serde_core",
"dep:serde_json",
]
ffi = ["bitflags"]
serde = [
"dep:serde_core",
"dep:serde",
]
[lib]
name = "arrow_schema"
path = "src/lib.rs"
bench = false
[[bench]]
name = "ffi"
path = "benches/ffi.rs"
harness = false
required-features = ["ffi"]
[dependencies.bitflags]
version = "2.0.0"
optional = true
default-features = false
[dependencies.serde]
version = "1.0"
features = ["derive"]
optional = true
default-features = false
[dependencies.serde_core]
version = "1.0"
features = [
"std",
"rc",
]
optional = true
default-features = false
[dependencies.serde_json]
version = "1.0"
optional = true
[dev-dependencies.bincode]
version = "2.0.1"
features = [
"std",
"serde",
]
default-features = false
[dev-dependencies.criterion]
version = "0.8.0"
default-features = false
[dev-dependencies.insta]
version = "1.43.1"
notes-service/vendor/arrow-schema/Cargo.toml.orig
Generated Vendored
+66
View File
@@ -0,0 +1,66 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
[package]
name = "arrow-schema"
version = { workspace = true }
description = "Defines the logical types for arrow arrays"
homepage = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }
keywords = { workspace = true }
include = { workspace = true }
edition = { workspace = true }
rust-version = { workspace = true }
[lib]
name = "arrow_schema"
bench = false
[dependencies]
serde_core = { version = "1.0", default-features = false, features = [
"std",
"rc",
], optional = true }
serde = { version = "1.0", default-features = false, features = [
"derive",
], optional = true }
bitflags = { version = "2.0.0", default-features = false, optional = true }
serde_json = { version = "1.0", optional = true }
[features]
canonical_extension_types = ["dep:serde_core", "dep:serde_json"]
# Enable ffi support
ffi = ["bitflags"]
serde = ["dep:serde_core", "dep:serde"]
[package.metadata.docs.rs]
all-features = true
[dev-dependencies]
bincode = { version = "2.0.1", default-features = false, features = [
"std",
"serde",
] }
criterion = { workspace = true, default-features = false }
insta = "1.43.1"
[[bench]]
name = "ffi"
harness = false
required-features = ["ffi"]
notes-service/vendor/arrow-schema/LICENSE.txt
Vendored
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
notes-service/vendor/arrow-schema/NOTICE.txt
Vendored
+84
View File
@@ -0,0 +1,84 @@
Apache Arrow
Copyright 2016-2019 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
This product includes software from the SFrame project (BSD, 3-clause).
* Copyright (C) 2015 Dato, Inc.
* Copyright (c) 2009 Carnegie Mellon University.
This product includes software from the Feather project (Apache 2.0)
https://github.com/wesm/feather
This product includes software from the DyND project (BSD 2-clause)
https://github.com/libdynd
This product includes software from the LLVM project
* distributed under the University of Illinois Open Source
This product includes software from the google-lint project
* Copyright (c) 2009 Google Inc. All rights reserved.
This product includes software from the mman-win32 project
* Copyright https://code.google.com/p/mman-win32/
* Licensed under the MIT License;
This product includes software from the LevelDB project
* Copyright (c) 2011 The LevelDB Authors. All rights reserved.
* Use of this source code is governed by a BSD-style license that can be
* Moved from Kudu http://github.com/cloudera/kudu
This product includes software from the CMake project
* Copyright 2001-2009 Kitware, Inc.
* Copyright 2012-2014 Continuum Analytics, Inc.
* All rights reserved.
This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause)
* Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved.
This product includes software from the Ibis project (Apache 2.0)
* Copyright (c) 2015 Cloudera, Inc.
* https://github.com/cloudera/ibis
This product includes software from Dremio (Apache 2.0)
* Copyright (C) 2017-2018 Dremio Corporation
* https://github.com/dremio/dremio-oss
This product includes software from Google Guava (Apache 2.0)
* Copyright (C) 2007 The Guava Authors
* https://github.com/google/guava
This product include software from CMake (BSD 3-Clause)
* CMake - Cross Platform Makefile Generator
* Copyright 2000-2019 Kitware, Inc. and Contributors
The web site includes files generated by Jekyll.
--------------------------------------------------------------------------------
This product includes code from Apache Kudu, which includes the following in
its NOTICE file:
Apache Kudu
Copyright 2016 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
Portions of this software were developed at
Cloudera, Inc (http://www.cloudera.com/).
--------------------------------------------------------------------------------
This product includes code from Apache ORC, which includes the following in
its NOTICE file:
Apache ORC
Copyright 2013-2019 The Apache Software Foundation
This product includes software developed by The Apache Software
Foundation (http://www.apache.org/).
This product includes software developed by Hewlett-Packard:
(c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P
notes-service/vendor/arrow-schema/benches/ffi.rs
Vendored
+38
View File
@@ -0,0 +1,38 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use arrow_schema::ffi::FFI_ArrowSchema;
use arrow_schema::{DataType, Field};
use criterion::*;
use std::sync::Arc;
fn criterion_benchmark(c: &mut Criterion) {
let fields = vec![
Arc::new(Field::new("c1", DataType::Utf8, false)),
Arc::new(Field::new("c2", DataType::Utf8, false)),
Arc::new(Field::new("c3", DataType::Utf8, false)),
Arc::new(Field::new("c4", DataType::Utf8, false)),
Arc::new(Field::new("c5", DataType::Utf8, false)),
];
let data_type = DataType::Struct(fields.into());
c.bench_function("ffi_arrow_schema_try_from", |b| {
b.iter(|| FFI_ArrowSchema::try_from(&data_type));
});
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);
notes-service/vendor/arrow-schema/src/datatype.rs
Vendored
+1242
View File
File diff suppressed because it is too large Load Diff
notes-service/vendor/arrow-schema/src/datatype_display.rs
Vendored
+556
View File
@@ -0,0 +1,556 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
use crate::DataType;
use std::fmt::Display;
use std::{collections::HashMap, fmt};
impl Display for DataType {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fn format_metadata(metadata: &HashMap<String, String>) -> String {
format!("{}", FormatMetadata(metadata))
}
fn format_nullability(field: &crate::Field) -> &str {
if field.is_nullable() { "" } else { "non-null " }
}
fn format_field(field: &crate::Field) -> String {
let name = field.name();
let maybe_nullable = format_nullability(field);
let data_type = field.data_type();
let metadata_str = format_metadata(field.metadata());
format!("{name:?}: {maybe_nullable}{data_type}{metadata_str}")
}
// A lot of these can still be improved a lot.
// _Some_ of these can be parsed with `FromStr`, but not all (YET!).
// The goal is that the formatting should always be
// * Terse and teadable
// * Reversible (contain all necessary information to reverse it perfectly)
match &self {
Self::Null => write!(f, "Null"),
Self::Boolean => write!(f, "Boolean"),
Self::Int8 => write!(f, "Int8"),
Self::Int16 => write!(f, "Int16"),
Self::Int32 => write!(f, "Int32"),
Self::Int64 => write!(f, "Int64"),
Self::UInt8 => write!(f, "UInt8"),
Self::UInt16 => write!(f, "UInt16"),
Self::UInt32 => write!(f, "UInt32"),
Self::UInt64 => write!(f, "UInt64"),
Self::Float16 => write!(f, "Float16"),
Self::Float32 => write!(f, "Float32"),
Self::Float64 => write!(f, "Float64"),
Self::Timestamp(time_unit, timezone) => {
if let Some(timezone) = timezone {
write!(f, "Timestamp({time_unit}, {timezone:?})")
} else {
write!(f, "Timestamp({time_unit})")
}
}
Self::Date32 => write!(f, "Date32"),
Self::Date64 => write!(f, "Date64"),
Self::Time32(time_unit) => write!(f, "Time32({time_unit})"),
Self::Time64(time_unit) => write!(f, "Time64({time_unit})"),
Self::Duration(time_unit) => write!(f, "Duration({time_unit})"),
Self::Interval(interval_unit) => write!(f, "Interval({interval_unit:?})"),
Self::Binary => write!(f, "Binary"),
Self::FixedSizeBinary(bytes_per_value) => {
write!(f, "FixedSizeBinary({bytes_per_value:?})")
}
Self::LargeBinary => write!(f, "LargeBinary"),
Self::BinaryView => write!(f, "BinaryView"),
Self::Utf8 => write!(f, "Utf8"),
Self::LargeUtf8 => write!(f, "LargeUtf8"),
Self::Utf8View => write!(f, "Utf8View"),
Self::List(field)
| Self::LargeList(field)
| Self::ListView(field)
| Self::LargeListView(field) => {
let type_name = if matches!(self, Self::List(_)) {
"List"
} else if matches!(self, Self::ListView(_)) {
"ListView"
} else if matches!(self, Self::LargeList(_)) {
"LargeList"
} else {
"LargeListView"
};
let name = field.name();
let maybe_nullable = format_nullability(field);
let data_type = field.data_type();
let field_name_str = if name == "item" {
String::default()
} else {
format!(", field: '{name}'")
};
let metadata_str = format_metadata(field.metadata());
// e.g. `LargeList(non-null Uint32)
write!(
f,
"{type_name}({maybe_nullable}{data_type}{field_name_str}{metadata_str})"
)
}
Self::FixedSizeList(field, size) => {
let name = field.name();
let maybe_nullable = format_nullability(field);
let data_type = field.data_type();
let field_name_str = if name == "item" {
String::default()
} else {
format!(", field: '{name}'")
};
let metadata_str = format_metadata(field.metadata());
write!(
f,
"FixedSizeList({size} x {maybe_nullable}{data_type}{field_name_str}{metadata_str})",
)
}
Self::Struct(fields) => {
write!(f, "Struct(")?;
if !fields.is_empty() {
let fields_str = fields
.iter()
.map(|field| format_field(field))
.collect::<Vec<_>>()
.join(", ");
write!(f, "{fields_str}")?;
}
write!(f, ")")?;
Ok(())
}
Self::Union(union_fields, union_mode) => {
write!(f, "Union({union_mode:?}")?;
if !union_fields.is_empty() {
write!(f, ", ")?;
let fields_str = union_fields
.iter()
.map(|v| {
let type_id = v.0;
let field_str = format_field(v.1);
format!("{type_id:?}: ({field_str})")
})
.collect::<Vec<_>>()
.join(", ");
write!(f, "{fields_str}")?;
}
write!(f, ")")?;
Ok(())
}
Self::Dictionary(data_type, data_type1) => {
write!(f, "Dictionary({data_type}, {data_type1})")
}
Self::Decimal32(precision, scale) => write!(f, "Decimal32({precision}, {scale})"),
Self::Decimal64(precision, scale) => write!(f, "Decimal64({precision}, {scale})"),
Self::Decimal128(precision, scale) => write!(f, "Decimal128({precision}, {scale})"),
Self::Decimal256(precision, scale) => write!(f, "Decimal256({precision}, {scale})"),
Self::Map(field, sorted) => {
write!(f, "Map(")?;
let map_field_str = format_field(field);
let keys_are_sorted = if *sorted { "sorted" } else { "unsorted" };
write!(f, "{map_field_str}, {keys_are_sorted})")?;
Ok(())
}
Self::RunEndEncoded(run_ends_field, values_field) => {
write!(f, "RunEndEncoded(")?;
let run_ends_str = format_field(run_ends_field);
let values_str = format_field(values_field);
write!(f, "{run_ends_str}, {values_str})")?;
Ok(())
}
}
}
}
/// Adapter to format a metadata HashMap consistently.
struct FormatMetadata<'a>(&'a HashMap<String, String>);
impl fmt::Display for FormatMetadata<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let metadata = self.0;
if metadata.is_empty() {
Ok(())
} else {
let mut entries: Vec<(&String, &String)> = metadata.iter().collect();
entries.sort_by(|a, b| a.0.cmp(b.0));
write!(f, ", metadata: ")?;
f.debug_map().entries(entries).finish()
}
}
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use crate::Field;
use super::*;
#[test]
fn test_display_list() {
let list_data_type = DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true)));
let list_data_type_string = list_data_type.to_string();
let expected_string = "List(Int32)";
assert_eq!(list_data_type_string, expected_string);
}
#[test]
fn test_display_list_view() {
let list_view_data_type =
DataType::ListView(Arc::new(Field::new("item", DataType::Int32, true)));
let list_view_data_type_string = list_view_data_type.to_string();
let expected_string = "ListView(Int32)";
assert_eq!(list_view_data_type_string, expected_string);
}
#[test]
fn test_display_list_with_named_field() {
let list_data_type = DataType::List(Arc::new(Field::new("foo", DataType::UInt64, false)));
let list_data_type_string = list_data_type.to_string();
let expected_string = "List(non-null UInt64, field: 'foo')";
assert_eq!(list_data_type_string, expected_string);
}
#[test]
fn test_display_list_view_with_named_field() {
let list_view_data_type =
DataType::ListView(Arc::new(Field::new("bar", DataType::UInt64, false)));
let list_view_data_type_string = list_view_data_type.to_string();
let expected_string = "ListView(non-null UInt64, field: 'bar')";
assert_eq!(list_view_data_type_string, expected_string);
}
#[test]
fn test_display_nested_list() {
let nested_data_type = DataType::List(Arc::new(Field::new_list_field(
DataType::List(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
)));
let nested_data_type_string = nested_data_type.to_string();
let nested_expected_string = "List(non-null List(non-null UInt64))";
assert_eq!(nested_data_type_string, nested_expected_string);
}
#[test]
fn test_display_nested_list_view() {
let nested_view_data_type = DataType::ListView(Arc::new(Field::new_list_field(
DataType::ListView(Arc::new(Field::new_list_field(DataType::UInt64, false))),
false,
)));
let nested_view_data_type_string = nested_view_data_type.to_string();
let nested_view_expected_string = "ListView(non-null ListView(non-null UInt64))";
assert_eq!(nested_view_data_type_string, nested_view_expected_string);
}
#[test]
fn test_display_list_with_metadata() {
let mut field = Field::new_list_field(DataType::Int32, true);
let metadata = HashMap::from([("foo1".to_string(), "value1".to_string())]);
field.set_metadata(metadata);
let list_data_type = DataType::List(Arc::new(field));
let list_data_type_string = list_data_type.to_string();
let expected_string = "List(Int32, metadata: {\"foo1\": \"value1\"})";
assert_eq!(list_data_type_string, expected_string);
}
#[test]
fn test_display_list_view_with_metadata() {
let mut field = Field::new_list_field(DataType::Int32, true);
let metadata = HashMap::from([("foo2".to_string(), "value2".to_string())]);
field.set_metadata(metadata);
let list_view_data_type = DataType::ListView(Arc::new(field));
let list_view_data_type_string = list_view_data_type.to_string();
let expected_string = "ListView(Int32, metadata: {\"foo2\": \"value2\"})";
assert_eq!(list_view_data_type_string, expected_string);
}
#[test]
fn test_display_large_list() {
let large_list_data_type =
DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int32, true)));
let large_list_data_type_string = large_list_data_type.to_string();
let expected_string = "LargeList(Int32)";
assert_eq!(large_list_data_type_string, expected_string);
// Test with named field
let large_list_named =
DataType::LargeList(Arc::new(Field::new("bar", DataType::UInt64, false)));
let large_list_named_string = large_list_named.to_string();
let expected_named_string = "LargeList(non-null UInt64, field: 'bar')";
assert_eq!(large_list_named_string, expected_named_string);
// Test with metadata
let mut field = Field::new_list_field(DataType::Int32, true);
let metadata = HashMap::from([("key1".to_string(), "value1".to_string())]);
field.set_metadata(metadata);
let large_list_metadata = DataType::LargeList(Arc::new(field));
let large_list_metadata_string = large_list_metadata.to_string();
let expected_metadata_string = "LargeList(Int32, metadata: {\"key1\": \"value1\"})";
assert_eq!(large_list_metadata_string, expected_metadata_string);
}
#[test]
fn test_display_large_list_view() {
let large_list_view_data_type =
DataType::LargeListView(Arc::new(Field::new("item", DataType::Int32, true)));
let large_list_view_data_type_string = large_list_view_data_type.to_string();
let expected_string = "LargeListView(Int32)";
assert_eq!(large_list_view_data_type_string, expected_string);
// Test with named field
let large_list_view_named =
DataType::LargeListView(Arc::new(Field::new("bar", DataType::UInt64, false)));
let large_list_view_named_string = large_list_view_named.to_string();
let expected_named_string = "LargeListView(non-null UInt64, field: 'bar')";
assert_eq!(large_list_view_named_string, expected_named_string);
// Test with metadata
let mut field = Field::new_list_field(DataType::Int32, true);
let metadata = HashMap::from([("key1".to_string(), "value1".to_string())]);
field.set_metadata(metadata);
let large_list_view_metadata = DataType::LargeListView(Arc::new(field));
let large_list_view_metadata_string = large_list_view_metadata.to_string();
let expected_metadata_string = "LargeListView(Int32, metadata: {\"key1\": \"value1\"})";
assert_eq!(large_list_view_metadata_string, expected_metadata_string);
}
#[test]
fn test_display_fixed_size_list() {
let fixed_size_list =
DataType::FixedSizeList(Arc::new(Field::new_list_field(DataType::Int32, true)), 5);
let fixed_size_list_string = fixed_size_list.to_string();
let expected_string = "FixedSizeList(5 x Int32)";
assert_eq!(fixed_size_list_string, expected_string);
// Test with named field
let fixed_size_named =
DataType::FixedSizeList(Arc::new(Field::new("baz", DataType::UInt64, false)), 3);
let fixed_size_named_string = fixed_size_named.to_string();
let expected_named_string = "FixedSizeList(3 x non-null UInt64, field: 'baz')";
assert_eq!(fixed_size_named_string, expected_named_string);
// Test with metadata
let mut field = Field::new_list_field(DataType::Int32, true);
let metadata = HashMap::from([("key2".to_string(), "value2".to_string())]);
field.set_metadata(metadata);
let fixed_size_metadata = DataType::FixedSizeList(Arc::new(field), 4);
let fixed_size_metadata_string = fixed_size_metadata.to_string();
let expected_metadata_string = "FixedSizeList(4 x Int32, metadata: {\"key2\": \"value2\"})";
assert_eq!(fixed_size_metadata_string, expected_metadata_string);
}
#[test]
fn test_display_struct() {
let fields = vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Utf8, true),
];
let struct_data_type = DataType::Struct(fields.into());
let struct_data_type_string = struct_data_type.to_string();
let expected_string = "Struct(\"a\": non-null Int32, \"b\": Utf8)";
assert_eq!(struct_data_type_string, expected_string);
// Test with metadata
let mut field_with_metadata = Field::new("b", DataType::Utf8, true);
let metadata = HashMap::from([
("key".to_string(), "value".to_string()),
("key2".to_string(), "value2".to_string()),
]);
field_with_metadata.set_metadata(metadata);
let struct_fields_with_metadata =
vec![Field::new("a", DataType::Int32, false), field_with_metadata];
let struct_data_type_with_metadata = DataType::Struct(struct_fields_with_metadata.into());
let struct_data_type_with_metadata_string = struct_data_type_with_metadata.to_string();
let expected_string_with_metadata = "Struct(\"a\": non-null Int32, \"b\": Utf8, metadata: {\"key\": \"value\", \"key2\": \"value2\"})";
assert_eq!(
struct_data_type_with_metadata_string,
expected_string_with_metadata
);
}
#[test]
fn test_display_union() {
let fields = vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Utf8, true),
];
let type_ids = vec![0, 1];
let union_fields = type_ids
.into_iter()
.zip(fields.into_iter().map(Arc::new))
.collect();
let union_data_type = DataType::Union(union_fields, crate::UnionMode::Sparse);
let union_data_type_string = union_data_type.to_string();
let expected_string = "Union(Sparse, 0: (\"a\": non-null Int32), 1: (\"b\": Utf8))";
assert_eq!(union_data_type_string, expected_string);
// Test with metadata
let mut field_with_metadata = Field::new("b", DataType::Utf8, true);
let metadata = HashMap::from([("key".to_string(), "value".to_string())]);
field_with_metadata.set_metadata(metadata);
let union_fields_with_metadata = vec![
(0, Arc::new(Field::new("a", DataType::Int32, false))),
(1, Arc::new(field_with_metadata)),
]
.into_iter()
.collect();
let union_data_type_with_metadata =
DataType::Union(union_fields_with_metadata, crate::UnionMode::Sparse);
let union_data_type_with_metadata_string = union_data_type_with_metadata.to_string();
let expected_string_with_metadata = "Union(Sparse, 0: (\"a\": non-null Int32), 1: (\"b\": Utf8, metadata: {\"key\": \"value\"}))";
assert_eq!(
union_data_type_with_metadata_string,
expected_string_with_metadata
);
}
#[test]
fn test_display_map() {
let entry_field = Field::new(
"entries",
DataType::Struct(
vec![
Field::new("key", DataType::Utf8, false),
Field::new("value", DataType::Int32, true),
]
.into(),
),
false,
);
let map_data_type = DataType::Map(Arc::new(entry_field), true);
let map_data_type_string = map_data_type.to_string();
let expected_string =
"Map(\"entries\": non-null Struct(\"key\": non-null Utf8, \"value\": Int32), sorted)";
assert_eq!(map_data_type_string, expected_string);
// Test with metadata
let mut entry_field_with_metadata = Field::new(
"entries",
DataType::Struct(
vec![
Field::new("key", DataType::Utf8, false),
Field::new("value", DataType::Int32, true),
]
.into(),
),
false,
);
let metadata = HashMap::from([("key".to_string(), "value".to_string())]);
entry_field_with_metadata.set_metadata(metadata);
let map_data_type_with_metadata = DataType::Map(Arc::new(entry_field_with_metadata), true);
let map_data_type_with_metadata_string = map_data_type_with_metadata.to_string();
let expected_string_with_metadata = "Map(\"entries\": non-null Struct(\"key\": non-null Utf8, \"value\": Int32), metadata: {\"key\": \"value\"}, sorted)";
assert_eq!(
map_data_type_with_metadata_string,
expected_string_with_metadata
);
}
#[test]
fn test_display_run_end_encoded() {
let run_ends_field = Arc::new(Field::new("run_ends", DataType::UInt32, false));
let values_field = Arc::new(Field::new("values", DataType::Int32, true));
let ree_data_type = DataType::RunEndEncoded(run_ends_field.clone(), values_field.clone());
let ree_data_type_string = ree_data_type.to_string();
let expected_string = "RunEndEncoded(\"run_ends\": non-null UInt32, \"values\": Int32)";
assert_eq!(ree_data_type_string, expected_string);
// Test with metadata
let mut run_ends_field_with_metadata = Field::new("run_ends", DataType::UInt32, false);
let metadata = HashMap::from([("key".to_string(), "value".to_string())]);
run_ends_field_with_metadata.set_metadata(metadata);
let ree_data_type_with_metadata =
DataType::RunEndEncoded(Arc::new(run_ends_field_with_metadata), values_field.clone());
let ree_data_type_with_metadata_string = ree_data_type_with_metadata.to_string();
let expected_string_with_metadata = "RunEndEncoded(\"run_ends\": non-null UInt32, metadata: {\"key\": \"value\"}, \"values\": Int32)";
assert_eq!(
ree_data_type_with_metadata_string,
expected_string_with_metadata
);
}
#[test]
fn test_display_dictionary() {
let dict_data_type =
DataType::Dictionary(Box::new(DataType::Int8), Box::new(DataType::Utf8));
let dict_data_type_string = dict_data_type.to_string();
let expected_string = "Dictionary(Int8, Utf8)";
assert_eq!(dict_data_type_string, expected_string);
// Test with complex index and value types
let complex_dict_data_type = DataType::Dictionary(
Box::new(DataType::Int16),
Box::new(DataType::Struct(
vec![
Field::new("a", DataType::Int32, false),
Field::new("b", DataType::Utf8, true),
]
.into(),
)),
);
let complex_dict_data_type_string = complex_dict_data_type.to_string();
let expected_complex_string =
"Dictionary(Int16, Struct(\"a\": non-null Int32, \"b\": Utf8))";
assert_eq!(complex_dict_data_type_string, expected_complex_string);
}
#[test]
fn test_display_interval() {
let interval_year_month = DataType::Interval(crate::IntervalUnit::YearMonth);
let interval_year_month_string = interval_year_month.to_string();
let expected_year_month_string = "Interval(YearMonth)";
assert_eq!(interval_year_month_string, expected_year_month_string);
let interval_day_time = DataType::Interval(crate::IntervalUnit::DayTime);
let interval_day_time_string = interval_day_time.to_string();
let expected_day_time_string = "Interval(DayTime)";
assert_eq!(interval_day_time_string, expected_day_time_string);
let interval_month_day_nano = DataType::Interval(crate::IntervalUnit::MonthDayNano);
let interval_month_day_nano_string = interval_month_day_nano.to_string();
let expected_month_day_nano_string = "Interval(MonthDayNano)";
assert_eq!(
interval_month_day_nano_string,
expected_month_day_nano_string
);
}
#[test]
fn test_display_timestamp() {
let timestamp_without_tz = DataType::Timestamp(crate::TimeUnit::Microsecond, None);
let timestamp_without_tz_string = timestamp_without_tz.to_string();
let expected_without_tz_string = "Timestamp(µs)";
assert_eq!(timestamp_without_tz_string, expected_without_tz_string);
let timestamp_with_tz =
DataType::Timestamp(crate::TimeUnit::Nanosecond, Some(Arc::from("UTC")));
let timestamp_with_tz_string = timestamp_with_tz.to_string();
let expected_with_tz_string = "Timestamp(ns, \"UTC\")";
assert_eq!(timestamp_with_tz_string, expected_with_tz_string);
}
}
notes-service/vendor/arrow-schema/src/datatype_parse.rs
Vendored
+1510
View File
File diff suppressed because it is too large Load Diff
notes-service/vendor/arrow-schema/src/error.rs
Vendored
+179
View File
@@ -0,0 +1,179 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Defines `ArrowError` for representing failures in various Arrow operations.
use std::fmt::{Debug, Display, Formatter};
use std::io::Write;
use std::error::Error;
/// Many different operations in the `arrow` crate return this error type.
#[derive(Debug)]
pub enum ArrowError {
/// Returned when functionality is not yet available.
NotYetImplemented(String),
/// Wraps an external error.
ExternalError(Box<dyn Error + Send + Sync>),
/// Error during casting from one type to another.
CastError(String),
/// Memory or buffer error.
MemoryError(String),
/// Error during parsing from a string.
ParseError(String),
/// Error during schema-related operations.
SchemaError(String),
/// Error during computation.
ComputeError(String),
/// Error during division by zero.
DivideByZero,
/// Error when an arithmetic operation overflows.
ArithmeticOverflow(String),
/// Error during CSV-related operations.
CsvError(String),
/// Error during JSON-related operations.
JsonError(String),
/// Error during Avro-related operations.
AvroError(String),
/// Error during IO operations.
IoError(String, std::io::Error),
/// Error during IPC operations in `arrow-ipc` or `arrow-flight`.
IpcError(String),
/// Error indicating that an unexpected or bad argument was passed to a function.
InvalidArgumentError(String),
/// Error during Parquet operations.
ParquetError(String),
/// Error during import or export to/from the C Data Interface
CDataInterface(String),
/// Error when a dictionary key is bigger than the key type
DictionaryKeyOverflowError,
/// Error when the run end index in a REE array is bigger than the array length
RunEndIndexOverflowError,
/// Error when the offset overflows.
OffsetOverflowError(usize),
}
impl ArrowError {
/// Wraps an external error in an `ArrowError`.
pub fn from_external_error(error: Box<dyn Error + Send + Sync>) -> Self {
Self::ExternalError(error)
}
}
impl From<std::io::Error> for ArrowError {
fn from(error: std::io::Error) -> Self {
ArrowError::IoError(error.to_string(), error)
}
}
impl From<std::str::Utf8Error> for ArrowError {
fn from(error: std::str::Utf8Error) -> Self {
ArrowError::ParseError(error.to_string())
}
}
impl From<std::string::FromUtf8Error> for ArrowError {
fn from(error: std::string::FromUtf8Error) -> Self {
ArrowError::ParseError(error.to_string())
}
}
impl<W: Write> From<std::io::IntoInnerError<W>> for ArrowError {
fn from(error: std::io::IntoInnerError<W>) -> Self {
ArrowError::IoError(error.to_string(), error.into())
}
}
impl Display for ArrowError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
ArrowError::NotYetImplemented(source) => {
write!(f, "Not yet implemented: {}", &source)
}
ArrowError::ExternalError(source) => write!(f, "External error: {}", &source),
ArrowError::CastError(desc) => write!(f, "Cast error: {desc}"),
ArrowError::MemoryError(desc) => write!(f, "Memory error: {desc}"),
ArrowError::ParseError(desc) => write!(f, "Parser error: {desc}"),
ArrowError::SchemaError(desc) => write!(f, "Schema error: {desc}"),
ArrowError::ComputeError(desc) => write!(f, "Compute error: {desc}"),
ArrowError::ArithmeticOverflow(desc) => write!(f, "Arithmetic overflow: {desc}"),
ArrowError::DivideByZero => write!(f, "Divide by zero error"),
ArrowError::AvroError(desc) => write!(f, "Avro error: {desc}"),
ArrowError::CsvError(desc) => write!(f, "Csv error: {desc}"),
ArrowError::JsonError(desc) => write!(f, "Json error: {desc}"),
ArrowError::IoError(desc, _) => write!(f, "Io error: {desc}"),
ArrowError::IpcError(desc) => write!(f, "Ipc error: {desc}"),
ArrowError::InvalidArgumentError(desc) => {
write!(f, "Invalid argument error: {desc}")
}
ArrowError::ParquetError(desc) => {
write!(f, "Parquet argument error: {desc}")
}
ArrowError::CDataInterface(desc) => {
write!(f, "C Data interface error: {desc}")
}
ArrowError::DictionaryKeyOverflowError => {
write!(f, "Dictionary key bigger than the key type")
}
ArrowError::RunEndIndexOverflowError => {
write!(f, "Run end encoded array index overflow error")
}
ArrowError::OffsetOverflowError(offset) => {
write!(f, "Offset overflow error: {offset}")
}
}
}
}
impl Error for ArrowError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
ArrowError::ExternalError(source) => Some(source.as_ref()),
ArrowError::IoError(_, source) => Some(source),
_ => None,
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn error_source() {
let e1 = ArrowError::DivideByZero;
assert!(e1.source().is_none());
// one level of wrapping
let e2 = ArrowError::ExternalError(Box::new(e1));
let source = e2.source().unwrap().downcast_ref::<ArrowError>().unwrap();
assert!(matches!(source, ArrowError::DivideByZero));
// two levels of wrapping
let e3 = ArrowError::ExternalError(Box::new(e2));
let source = e3
.source()
.unwrap()
.downcast_ref::<ArrowError>()
.unwrap()
.source()
.unwrap()
.downcast_ref::<ArrowError>()
.unwrap();
assert!(matches!(source, ArrowError::DivideByZero));
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/bool8.rs
Vendored
+142
View File
@@ -0,0 +1,142 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! 8-bit Boolean
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#bit-boolean>
use crate::{ArrowError, DataType, extension::ExtensionType};
/// The extension type for `8-bit Boolean`.
///
/// Extension name: `arrow.bool8`.
///
/// The storage type of the extension is `Int8` where:
/// - false is denoted by the value 0.
/// - true can be specified using any non-zero value. Preferably 1.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#bit-boolean>
#[derive(Debug, Default, Clone, Copy, PartialEq)]
pub struct Bool8;
impl ExtensionType for Bool8 {
const NAME: &'static str = "arrow.bool8";
type Metadata = &'static str;
fn metadata(&self) -> &Self::Metadata {
&""
}
fn serialize_metadata(&self) -> Option<String> {
Some(String::default())
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
if metadata.is_some_and(str::is_empty) {
Ok("")
} else {
Err(ArrowError::InvalidArgumentError(
"Bool8 extension type expects an empty string as metadata".to_owned(),
))
}
}
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
match data_type {
DataType::Int8 => Ok(()),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"Bool8 data type mismatch, expected Int8, found {data_type}"
))),
}
}
fn try_new(data_type: &DataType, _metadata: Self::Metadata) -> Result<Self, ArrowError> {
Self.supports_data_type(data_type).map(|_| Self)
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let mut field = Field::new("", DataType::Int8, false);
field.try_with_extension_type(Bool8)?;
field.try_extension_type::<Bool8>()?;
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::Bool8(Bool8)
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field = Field::new("", DataType::Int8, false).with_metadata(
[(EXTENSION_TYPE_METADATA_KEY.to_owned(), "".to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<Bool8>();
}
#[test]
#[should_panic(expected = "expected Int8, found Boolean")]
fn invalid_type() {
Field::new("", DataType::Boolean, false).with_extension_type(Bool8);
}
#[test]
#[should_panic(expected = "Bool8 extension type expects an empty string as metadata")]
fn missing_metadata() {
let field = Field::new("", DataType::Int8, false).with_metadata(
[(EXTENSION_TYPE_NAME_KEY.to_owned(), Bool8::NAME.to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<Bool8>();
}
#[test]
#[should_panic(expected = "Bool8 extension type expects an empty string as metadata")]
fn invalid_metadata() {
let field = Field::new("", DataType::Int8, false).with_metadata(
[
(EXTENSION_TYPE_NAME_KEY.to_owned(), Bool8::NAME.to_owned()),
(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
"non-empty".to_owned(),
),
]
.into_iter()
.collect(),
);
field.extension_type::<Bool8>();
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/fixed_shape_tensor.rs
Vendored
+582
View File
@@ -0,0 +1,582 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! FixedShapeTensor
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#fixed-shape-tensor>
use serde_core::de::{self, MapAccess, Visitor};
use serde_core::ser::SerializeStruct;
use serde_core::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use crate::{ArrowError, DataType, extension::ExtensionType};
/// The extension type for fixed shape tensor.
///
/// Extension name: `arrow.fixed_shape_tensor`.
///
/// The storage type of the extension: `FixedSizeList` where:
/// - `value_type` is the data type of individual tensor elements.
/// - `list_size` is the product of all the elements in tensor shape.
///
/// Extension type parameters:
/// - `value_type`: the Arrow data type of individual tensor elements.
/// - `shape`: the physical shape of the contained tensors as an array.
///
/// Optional parameters describing the logical layout:
/// - `dim_names`: explicit names to tensor dimensions as an array. The
/// length of it should be equal to the shape length and equal to the
/// number of dimensions.
/// `dim_names` can be used if the dimensions have
/// well-known names and they map to the physical layout (row-major).
/// - `permutation`: indices of the desired ordering of the original
/// dimensions, defined as an array.
/// The indices contain a permutation of the values `[0, 1, .., N-1]`
/// where `N` is the number of dimensions. The permutation indicates
/// which dimension of the logical layout corresponds to which dimension
/// of the physical tensor (the i-th dimension of the logical view
/// corresponds to the dimension with number `permutations[i]` of the
/// physical tensor).
/// Permutation can be useful in case the logical order of the tensor is
/// a permutation of the physical order (row-major).
/// When logical and physical layout are equal, the permutation will
/// always be `([0, 1, .., N-1])` and can therefore be left out.
///
/// Description of the serialization:
/// The metadata must be a valid JSON object including shape of the
/// contained tensors as an array with key `shape` plus optional
/// dimension names with keys `dim_names` and ordering of the
/// dimensions with key `permutation`.
/// Example: `{ "shape": [2, 5]}`
/// Example with `dim_names` metadata for NCHW ordered data:
/// `{ "shape": [100, 200, 500], "dim_names": ["C", "H", "W"]}`
/// Example of permuted 3-dimensional tensor:
/// `{ "shape": [100, 200, 500], "permutation": [2, 0, 1]}`
///
/// This is the physical layout shape and the shape of the logical layout
/// would in this case be `[500, 100, 200]`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#fixed-shape-tensor>
#[derive(Debug, Clone, PartialEq)]
pub struct FixedShapeTensor {
/// The data type of individual tensor elements.
value_type: DataType,
/// The metadata of this extension type.
metadata: FixedShapeTensorMetadata,
}
impl FixedShapeTensor {
/// Returns a new fixed shape tensor extension type.
///
/// # Error
///
/// Return an error if the provided dimension names or permutations are
/// invalid.
pub fn try_new(
value_type: DataType,
shape: impl IntoIterator<Item = usize>,
dimension_names: Option<Vec<String>>,
permutations: Option<Vec<usize>>,
) -> Result<Self, ArrowError> {
// TODO: are all data types are suitable as value type?
FixedShapeTensorMetadata::try_new(shape, dimension_names, permutations).map(|metadata| {
Self {
value_type,
metadata,
}
})
}
/// Returns the value type of the individual tensor elements.
pub fn value_type(&self) -> &DataType {
&self.value_type
}
/// Returns the product of all the elements in tensor shape.
pub fn list_size(&self) -> usize {
self.metadata.list_size()
}
/// Returns the number of dimensions in this fixed shape tensor.
pub fn dimensions(&self) -> usize {
self.metadata.dimensions()
}
/// Returns the names of the dimensions in this fixed shape tensor, if
/// set.
pub fn dimension_names(&self) -> Option<&[String]> {
self.metadata.dimension_names()
}
/// Returns the indices of the desired ordering of the original
/// dimensions, if set.
pub fn permutations(&self) -> Option<&[usize]> {
self.metadata.permutations()
}
}
/// Extension type metadata for [`FixedShapeTensor`].
#[derive(Debug, Clone, PartialEq)]
pub struct FixedShapeTensorMetadata {
/// The physical shape of the contained tensors.
shape: Vec<usize>,
/// Explicit names to tensor dimensions.
dim_names: Option<Vec<String>>,
/// Indices of the desired ordering of the original dimensions.
permutations: Option<Vec<usize>>,
}
impl Serialize for FixedShapeTensorMetadata {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("FixedShapeTensorMetadata", 3)?;
state.serialize_field("shape", &self.shape)?;
state.serialize_field("dim_names", &self.dim_names)?;
state.serialize_field("permutations", &self.permutations)?;
state.end()
}
}
#[derive(Debug)]
enum MetadataField {
Shape,
DimNames,
Permutations,
}
struct MetadataFieldVisitor;
impl<'de> Visitor<'de> for MetadataFieldVisitor {
type Value = MetadataField;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("`shape`, `dim_names`, or `permutations`")
}
fn visit_str<E>(self, value: &str) -> Result<MetadataField, E>
where
E: de::Error,
{
match value {
"shape" => Ok(MetadataField::Shape),
"dim_names" => Ok(MetadataField::DimNames),
"permutations" => Ok(MetadataField::Permutations),
_ => Err(de::Error::unknown_field(
value,
&["shape", "dim_names", "permutations"],
)),
}
}
}
impl<'de> Deserialize<'de> for MetadataField {
fn deserialize<D>(deserializer: D) -> Result<MetadataField, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_identifier(MetadataFieldVisitor)
}
}
struct FixedShapeTensorMetadataVisitor;
impl<'de> Visitor<'de> for FixedShapeTensorMetadataVisitor {
type Value = FixedShapeTensorMetadata;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("struct FixedShapeTensorMetadata")
}
fn visit_seq<V>(self, mut seq: V) -> Result<FixedShapeTensorMetadata, V::Error>
where
V: de::SeqAccess<'de>,
{
let shape = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(0, &self))?;
let dim_names = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
let permutations = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(2, &self))?;
Ok(FixedShapeTensorMetadata {
shape,
dim_names,
permutations,
})
}
fn visit_map<V>(self, mut map: V) -> Result<FixedShapeTensorMetadata, V::Error>
where
V: MapAccess<'de>,
{
let mut shape = None;
let mut dim_names = None;
let mut permutations = None;
while let Some(key) = map.next_key()? {
match key {
MetadataField::Shape => {
if shape.is_some() {
return Err(de::Error::duplicate_field("shape"));
}
shape = Some(map.next_value()?);
}
MetadataField::DimNames => {
if dim_names.is_some() {
return Err(de::Error::duplicate_field("dim_names"));
}
dim_names = Some(map.next_value()?);
}
MetadataField::Permutations => {
if permutations.is_some() {
return Err(de::Error::duplicate_field("permutations"));
}
permutations = Some(map.next_value()?);
}
}
}
let shape = shape.ok_or_else(|| de::Error::missing_field("shape"))?;
Ok(FixedShapeTensorMetadata {
shape,
dim_names,
permutations,
})
}
}
impl<'de> Deserialize<'de> for FixedShapeTensorMetadata {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_struct(
"FixedShapeTensorMetadata",
&["shape", "dim_names", "permutations"],
FixedShapeTensorMetadataVisitor,
)
}
}
impl FixedShapeTensorMetadata {
/// Returns metadata for a fixed shape tensor extension type.
///
/// # Error
///
/// Return an error if the provided dimension names or permutations are
/// invalid.
pub fn try_new(
shape: impl IntoIterator<Item = usize>,
dimension_names: Option<Vec<String>>,
permutations: Option<Vec<usize>>,
) -> Result<Self, ArrowError> {
let shape = shape.into_iter().collect::<Vec<_>>();
let dimensions = shape.len();
let dim_names = dimension_names.map(|dimension_names| {
if dimension_names.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len()
)))
} else {
Ok(dimension_names)
}
}).transpose()?;
let permutations = permutations
.map(|permutations| {
if permutations.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations size mismatch, expected {dimensions}, found {}",
permutations.len()
)))
} else {
let mut sorted_permutations = permutations.clone();
sorted_permutations.sort_unstable();
if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}"
)))
} else {
Ok(permutations)
}
}
})
.transpose()?;
Ok(Self {
shape,
dim_names,
permutations,
})
}
/// Returns the product of all the elements in tensor shape.
pub fn list_size(&self) -> usize {
self.shape.iter().product()
}
/// Returns the number of dimensions in this fixed shape tensor.
pub fn dimensions(&self) -> usize {
self.shape.len()
}
/// Returns the names of the dimensions in this fixed shape tensor, if
/// set.
pub fn dimension_names(&self) -> Option<&[String]> {
self.dim_names.as_ref().map(AsRef::as_ref)
}
/// Returns the indices of the desired ordering of the original
/// dimensions, if set.
pub fn permutations(&self) -> Option<&[usize]> {
self.permutations.as_ref().map(AsRef::as_ref)
}
}
impl ExtensionType for FixedShapeTensor {
const NAME: &'static str = "arrow.fixed_shape_tensor";
type Metadata = FixedShapeTensorMetadata;
fn metadata(&self) -> &Self::Metadata {
&self.metadata
}
fn serialize_metadata(&self) -> Option<String> {
Some(serde_json::to_string(&self.metadata).expect("metadata serialization"))
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
metadata.map_or_else(
|| {
Err(ArrowError::InvalidArgumentError(
"FixedShapeTensor extension types requires metadata".to_owned(),
))
},
|value| {
serde_json::from_str(value).map_err(|e| {
ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor metadata deserialization failed: {e}"
))
})
},
)
}
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
let expected = DataType::new_fixed_size_list(
self.value_type.clone(),
i32::try_from(self.list_size()).expect("overflow"),
false,
);
data_type
.equals_datatype(&expected)
.then_some(())
.ok_or_else(|| {
ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor data type mismatch, expected {expected}, found {data_type}"
))
})
}
fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
match data_type {
DataType::FixedSizeList(field, list_size) if !field.is_nullable() => {
// Make sure the metadata is valid.
let metadata = FixedShapeTensorMetadata::try_new(
metadata.shape,
metadata.dim_names,
metadata.permutations,
)?;
// Make sure it is compatible with this data type.
let expected_size = i32::try_from(metadata.list_size()).expect("overflow");
if *list_size != expected_size {
Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor list size mismatch, expected {expected_size} (metadata), found {list_size} (data type)"
)))
} else {
Ok(Self {
value_type: field.data_type().clone(),
metadata,
})
}
}
data_type => Err(ArrowError::InvalidArgumentError(format!(
"FixedShapeTensor data type mismatch, expected FixedSizeList with non-nullable field, found {data_type}"
))),
}
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let fixed_shape_tensor = FixedShapeTensor::try_new(
DataType::Float32,
[100, 200, 500],
Some(vec!["C".to_owned(), "H".to_owned(), "W".to_owned()]),
Some(vec![2, 0, 1]),
)?;
let mut field = Field::new_fixed_size_list(
"",
Field::new("", DataType::Float32, false),
i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
false,
);
field.try_with_extension_type(fixed_shape_tensor.clone())?;
assert_eq!(
field.try_extension_type::<FixedShapeTensor>()?,
fixed_shape_tensor
);
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::FixedShapeTensor(fixed_shape_tensor)
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field =
Field::new_fixed_size_list("", Field::new("", DataType::Float32, false), 3, false)
.with_metadata(
[(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
r#"{ "shape": [100, 200, 500], }"#.to_owned(),
)]
.into_iter()
.collect(),
);
field.extension_type::<FixedShapeTensor>();
}
#[test]
#[should_panic(expected = "FixedShapeTensor data type mismatch, expected FixedSizeList")]
fn invalid_type() {
let fixed_shape_tensor =
FixedShapeTensor::try_new(DataType::Int32, [100, 200, 500], None, None).unwrap();
let field = Field::new_fixed_size_list(
"",
Field::new("", DataType::Float32, false),
i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
false,
);
field.with_extension_type(fixed_shape_tensor);
}
#[test]
#[should_panic(expected = "FixedShapeTensor extension types requires metadata")]
fn missing_metadata() {
let field =
Field::new_fixed_size_list("", Field::new("", DataType::Float32, false), 3, false)
.with_metadata(
[(
EXTENSION_TYPE_NAME_KEY.to_owned(),
FixedShapeTensor::NAME.to_owned(),
)]
.into_iter()
.collect(),
);
field.extension_type::<FixedShapeTensor>();
}
#[test]
#[should_panic(expected = "FixedShapeTensor metadata deserialization failed: \
unknown field `not-shape`, expected one of `shape`, `dim_names`, `permutations`")]
fn invalid_metadata() {
let fixed_shape_tensor =
FixedShapeTensor::try_new(DataType::Float32, [100, 200, 500], None, None).unwrap();
let field = Field::new_fixed_size_list(
"",
Field::new("", DataType::Float32, false),
i32::try_from(fixed_shape_tensor.list_size()).expect("overflow"),
false,
)
.with_metadata(
[
(
EXTENSION_TYPE_NAME_KEY.to_owned(),
FixedShapeTensor::NAME.to_owned(),
),
(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
r#"{ "not-shape": [] }"#.to_owned(),
),
]
.into_iter()
.collect(),
);
field.extension_type::<FixedShapeTensor>();
}
#[test]
#[should_panic(
expected = "FixedShapeTensor dimension names size mismatch, expected 3, found 2"
)]
fn invalid_metadata_dimension_names() {
FixedShapeTensor::try_new(
DataType::Float32,
[100, 200, 500],
Some(vec!["a".to_owned(), "b".to_owned()]),
None,
)
.unwrap();
}
#[test]
#[should_panic(expected = "FixedShapeTensor permutations size mismatch, expected 3, found 2")]
fn invalid_metadata_permutations_len() {
FixedShapeTensor::try_new(DataType::Float32, [100, 200, 500], None, Some(vec![1, 0]))
.unwrap();
}
#[test]
#[should_panic(
expected = "FixedShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: 3"
)]
fn invalid_metadata_permutations_values() {
FixedShapeTensor::try_new(
DataType::Float32,
[100, 200, 500],
None,
Some(vec![4, 3, 2]),
)
.unwrap();
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/json.rs
Vendored
+269
View File
@@ -0,0 +1,269 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! JSON
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#json>
use serde_core::de::{self, MapAccess, Visitor};
use serde_core::ser::SerializeStruct;
use serde_core::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use crate::{ArrowError, DataType, extension::ExtensionType};
/// The extension type for `JSON`.
///
/// Extension name: `arrow.json`.
///
/// The storage type of this extension is `String` or `LargeString` or
/// `StringView`. Only UTF-8 encoded JSON as specified in [rfc8259](https://datatracker.ietf.org/doc/html/rfc8259)
/// is supported.
///
/// This type does not have any parameters.
///
/// Metadata is either an empty string or a JSON string with an empty
/// object. In the future, additional fields may be added, but they are not
/// required to interpret the array.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#json>
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Json(JsonMetadata);
/// Empty object
#[derive(Debug, Clone, Copy, PartialEq)]
struct Empty {}
impl Serialize for Empty {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let state = serializer.serialize_struct("Empty", 0)?;
state.end()
}
}
struct EmptyVisitor;
impl<'de> Visitor<'de> for EmptyVisitor {
type Value = Empty;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("struct Empty")
}
fn visit_seq<A>(self, mut _seq: A) -> Result<Self::Value, A::Error>
where
A: de::SeqAccess<'de>,
{
Ok(Empty {})
}
fn visit_map<V>(self, mut map: V) -> Result<Empty, V::Error>
where
V: MapAccess<'de>,
{
if let Some(key) = map.next_key::<String>()? {
return Err(de::Error::unknown_field(&key, EMPTY_FIELDS));
}
Ok(Empty {})
}
fn visit_u64<E>(self, _v: u64) -> Result<Self::Value, E>
where
E: de::Error,
{
Err(de::Error::unknown_field("", EMPTY_FIELDS))
}
fn visit_str<E>(self, _v: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
Err(de::Error::unknown_field("", EMPTY_FIELDS))
}
fn visit_bytes<E>(self, _v: &[u8]) -> Result<Self::Value, E>
where
E: de::Error,
{
Err(de::Error::unknown_field("", EMPTY_FIELDS))
}
}
static EMPTY_FIELDS: &[&str] = &[];
impl<'de> Deserialize<'de> for Empty {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_struct("Empty", EMPTY_FIELDS, EmptyVisitor)
}
}
/// Extension type metadata for [`Json`].
#[derive(Debug, Default, Clone, PartialEq)]
pub struct JsonMetadata(Option<Empty>);
impl ExtensionType for Json {
const NAME: &'static str = "arrow.json";
type Metadata = JsonMetadata;
fn metadata(&self) -> &Self::Metadata {
&self.0
}
fn serialize_metadata(&self) -> Option<String> {
Some(
self.metadata()
.0
.as_ref()
.map(serde_json::to_string)
.map(Result::unwrap)
.unwrap_or_else(|| "".to_owned()),
)
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
const ERR: &str = "Json extension type metadata is either an empty string or a JSON string with an empty object";
metadata
.map_or_else(
|| Err(ArrowError::InvalidArgumentError(ERR.to_owned())),
|metadata| {
match metadata {
// Empty string
"" => Ok(None),
value => serde_json::from_str::<Empty>(value)
.map(Option::Some)
.map_err(|_| ArrowError::InvalidArgumentError(ERR.to_owned())),
}
},
)
.map(JsonMetadata)
}
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
match data_type {
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Ok(()),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"Json data type mismatch, expected one of Utf8, LargeUtf8, Utf8View, found {data_type}"
))),
}
}
fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
let json = Self(metadata);
json.supports_data_type(data_type)?;
Ok(json)
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let mut field = Field::new("", DataType::Utf8, false);
field.try_with_extension_type(Json::default())?;
assert_eq!(
field.metadata().get(EXTENSION_TYPE_METADATA_KEY),
Some(&"".to_owned())
);
assert_eq!(
field.try_extension_type::<Json>()?,
Json(JsonMetadata(None))
);
let mut field = Field::new("", DataType::LargeUtf8, false);
field.try_with_extension_type(Json(JsonMetadata(Some(Empty {}))))?;
assert_eq!(
field.metadata().get(EXTENSION_TYPE_METADATA_KEY),
Some(&"{}".to_owned())
);
assert_eq!(
field.try_extension_type::<Json>()?,
Json(JsonMetadata(Some(Empty {})))
);
let mut field = Field::new("", DataType::Utf8View, false);
field.try_with_extension_type(Json::default())?;
field.try_extension_type::<Json>()?;
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::Json(Json::default())
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field = Field::new("", DataType::Int8, false).with_metadata(
[(EXTENSION_TYPE_METADATA_KEY.to_owned(), "{}".to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<Json>();
}
#[test]
#[should_panic(expected = "expected one of Utf8, LargeUtf8, Utf8View, found Null")]
fn invalid_type() {
Field::new("", DataType::Null, false).with_extension_type(Json::default());
}
#[test]
#[should_panic(
expected = "Json extension type metadata is either an empty string or a JSON string with an empty object"
)]
fn invalid_metadata() {
let field = Field::new("", DataType::Utf8, false).with_metadata(
[
(EXTENSION_TYPE_NAME_KEY.to_owned(), Json::NAME.to_owned()),
(EXTENSION_TYPE_METADATA_KEY.to_owned(), "1234".to_owned()),
]
.into_iter()
.collect(),
);
field.extension_type::<Json>();
}
#[test]
#[should_panic(
expected = "Json extension type metadata is either an empty string or a JSON string with an empty object"
)]
fn missing_metadata() {
let field = Field::new("", DataType::LargeUtf8, false).with_metadata(
[(EXTENSION_TYPE_NAME_KEY.to_owned(), Json::NAME.to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<Json>();
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/mod.rs
Vendored
+150
View File
@@ -0,0 +1,150 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Canonical extension types.
//!
//! The Arrow columnar format allows defining extension types so as to extend
//! standard Arrow data types with custom semantics. Often these semantics will
//! be specific to a system or application. However, it is beneficial to share
//! the definitions of well-known extension types so as to improve
//! interoperability between different systems integrating Arrow columnar data.
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#format-canonical-extensions>
mod bool8;
pub use bool8::Bool8;
mod fixed_shape_tensor;
pub use fixed_shape_tensor::{FixedShapeTensor, FixedShapeTensorMetadata};
mod json;
pub use json::{Json, JsonMetadata};
mod opaque;
pub use opaque::{Opaque, OpaqueMetadata};
mod uuid;
pub use uuid::Uuid;
mod variable_shape_tensor;
pub use variable_shape_tensor::{VariableShapeTensor, VariableShapeTensorMetadata};
use crate::{ArrowError, Field};
use super::ExtensionType;
/// Canonical extension types.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#format-canonical-extensions>
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq)]
pub enum CanonicalExtensionType {
/// The extension type for `FixedShapeTensor`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#fixed-shape-tensor>
FixedShapeTensor(FixedShapeTensor),
/// The extension type for `VariableShapeTensor`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#variable-shape-tensor>
VariableShapeTensor(VariableShapeTensor),
/// The extension type for 'JSON'.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#json>
Json(Json),
/// The extension type for `UUID`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid>
Uuid(Uuid),
/// The extension type for `Opaque`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#opaque>
Opaque(Opaque),
/// The extension type for `Bool8`.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#bit-boolean>
Bool8(Bool8),
}
impl TryFrom<&Field> for CanonicalExtensionType {
type Error = ArrowError;
fn try_from(value: &Field) -> Result<Self, Self::Error> {
// Canonical extension type names start with `arrow.`
match value.extension_type_name() {
// An extension type name with an `arrow.` prefix
Some(name) if name.starts_with("arrow.") => match name {
FixedShapeTensor::NAME => value
.try_extension_type::<FixedShapeTensor>()
.map(Into::into),
VariableShapeTensor::NAME => value
.try_extension_type::<VariableShapeTensor>()
.map(Into::into),
Json::NAME => value.try_extension_type::<Json>().map(Into::into),
Uuid::NAME => value.try_extension_type::<Uuid>().map(Into::into),
Opaque::NAME => value.try_extension_type::<Opaque>().map(Into::into),
Bool8::NAME => value.try_extension_type::<Bool8>().map(Into::into),
_ => Err(ArrowError::InvalidArgumentError(format!(
"Unsupported canonical extension type: {name}"
))),
},
// Name missing the expected prefix
Some(name) => Err(ArrowError::InvalidArgumentError(format!(
"Field extension type name mismatch, expected a name with an `arrow.` prefix, found {name}"
))),
// Name missing
None => Err(ArrowError::InvalidArgumentError(
"Field extension type name missing".to_owned(),
)),
}
}
}
impl From<FixedShapeTensor> for CanonicalExtensionType {
fn from(value: FixedShapeTensor) -> Self {
CanonicalExtensionType::FixedShapeTensor(value)
}
}
impl From<VariableShapeTensor> for CanonicalExtensionType {
fn from(value: VariableShapeTensor) -> Self {
CanonicalExtensionType::VariableShapeTensor(value)
}
}
impl From<Json> for CanonicalExtensionType {
fn from(value: Json) -> Self {
CanonicalExtensionType::Json(value)
}
}
impl From<Uuid> for CanonicalExtensionType {
fn from(value: Uuid) -> Self {
CanonicalExtensionType::Uuid(value)
}
}
impl From<Opaque> for CanonicalExtensionType {
fn from(value: Opaque) -> Self {
CanonicalExtensionType::Opaque(value)
}
}
impl From<Bool8> for CanonicalExtensionType {
fn from(value: Bool8) -> Self {
CanonicalExtensionType::Bool8(value)
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/opaque.rs
Vendored
+330
View File
@@ -0,0 +1,330 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Opaque
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#opaque>
use serde_core::ser::SerializeStruct;
use serde_core::{
Deserialize, Deserializer, Serialize, Serializer,
de::{MapAccess, Visitor},
};
use crate::{ArrowError, DataType, extension::ExtensionType};
/// The extension type for `Opaque`.
///
/// Extension name: `arrow.opaque`.
///
/// Opaque represents a type that an Arrow-based system received from an
/// external (often non-Arrow) system, but that it cannot interpret. In this
/// case, it can pass on Opaque to its clients to at least show that a field
/// exists and preserve metadata about the type from the other system.
///
/// The storage type of this extension is any type. If there is no underlying
/// data, the storage type should be Null.
#[derive(Debug, Clone, PartialEq)]
pub struct Opaque(OpaqueMetadata);
impl Opaque {
/// Returns a new `Opaque` extension type.
pub fn new(type_name: impl Into<String>, vendor_name: impl Into<String>) -> Self {
Self(OpaqueMetadata::new(type_name, vendor_name))
}
/// Returns the name of the unknown type in the external system.
pub fn type_name(&self) -> &str {
self.0.type_name()
}
/// Returns the name of the external system.
pub fn vendor_name(&self) -> &str {
self.0.vendor_name()
}
}
impl From<OpaqueMetadata> for Opaque {
fn from(value: OpaqueMetadata) -> Self {
Self(value)
}
}
/// Extension type metadata for [`Opaque`].
#[derive(Debug, Clone, PartialEq)]
pub struct OpaqueMetadata {
/// Name of the unknown type in the external system.
type_name: String,
/// Name of the external system.
vendor_name: String,
}
impl Serialize for OpaqueMetadata {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut state = serializer.serialize_struct("OpaqueMetadata", 2)?;
state.serialize_field("type_name", &self.type_name)?;
state.serialize_field("vendor_name", &self.vendor_name)?;
state.end()
}
}
#[derive(Debug)]
enum MetadataField {
TypeName,
VendorName,
}
struct MetadataFieldVisitor;
impl<'de> Visitor<'de> for MetadataFieldVisitor {
type Value = MetadataField;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("`type_name` or `vendor_name`")
}
fn visit_str<E>(self, value: &str) -> Result<MetadataField, E>
where
E: serde_core::de::Error,
{
match value {
"type_name" => Ok(MetadataField::TypeName),
"vendor_name" => Ok(MetadataField::VendorName),
_ => Err(serde_core::de::Error::unknown_field(
value,
&["type_name", "vendor_name"],
)),
}
}
}
impl<'de> Deserialize<'de> for MetadataField {
fn deserialize<D>(deserializer: D) -> Result<MetadataField, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_identifier(MetadataFieldVisitor)
}
}
struct OpaqueMetadataVisitor;
impl<'de> Visitor<'de> for OpaqueMetadataVisitor {
type Value = OpaqueMetadata;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("struct OpaqueMetadata")
}
fn visit_seq<V>(self, mut seq: V) -> Result<OpaqueMetadata, V::Error>
where
V: serde_core::de::SeqAccess<'de>,
{
let type_name = seq
.next_element()?
.ok_or_else(|| serde_core::de::Error::invalid_length(0, &self))?;
let vendor_name = seq
.next_element()?
.ok_or_else(|| serde_core::de::Error::invalid_length(1, &self))?;
Ok(OpaqueMetadata {
type_name,
vendor_name,
})
}
fn visit_map<V>(self, mut map: V) -> Result<OpaqueMetadata, V::Error>
where
V: MapAccess<'de>,
{
let mut type_name = None;
let mut vendor_name = None;
while let Some(key) = map.next_key()? {
match key {
MetadataField::TypeName => {
if type_name.is_some() {
return Err(serde_core::de::Error::duplicate_field("type_name"));
}
type_name = Some(map.next_value()?);
}
MetadataField::VendorName => {
if vendor_name.is_some() {
return Err(serde_core::de::Error::duplicate_field("vendor_name"));
}
vendor_name = Some(map.next_value()?);
}
}
}
let type_name =
type_name.ok_or_else(|| serde_core::de::Error::missing_field("type_name"))?;
let vendor_name =
vendor_name.ok_or_else(|| serde_core::de::Error::missing_field("vendor_name"))?;
Ok(OpaqueMetadata {
type_name,
vendor_name,
})
}
}
impl<'de> Deserialize<'de> for OpaqueMetadata {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_struct(
"OpaqueMetadata",
&["type_name", "vendor_name"],
OpaqueMetadataVisitor,
)
}
}
impl OpaqueMetadata {
/// Returns a new `OpaqueMetadata`.
pub fn new(type_name: impl Into<String>, vendor_name: impl Into<String>) -> Self {
OpaqueMetadata {
type_name: type_name.into(),
vendor_name: vendor_name.into(),
}
}
/// Returns the name of the unknown type in the external system.
pub fn type_name(&self) -> &str {
&self.type_name
}
/// Returns the name of the external system.
pub fn vendor_name(&self) -> &str {
&self.vendor_name
}
}
impl ExtensionType for Opaque {
const NAME: &'static str = "arrow.opaque";
type Metadata = OpaqueMetadata;
fn metadata(&self) -> &Self::Metadata {
&self.0
}
fn serialize_metadata(&self) -> Option<String> {
Some(serde_json::to_string(self.metadata()).expect("metadata serialization"))
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
metadata.map_or_else(
|| {
Err(ArrowError::InvalidArgumentError(
"Opaque extension types requires metadata".to_owned(),
))
},
|value| {
serde_json::from_str(value).map_err(|e| {
ArrowError::InvalidArgumentError(format!(
"Opaque metadata deserialization failed: {e}"
))
})
},
)
}
fn supports_data_type(&self, _data_type: &DataType) -> Result<(), ArrowError> {
// Any type
Ok(())
}
fn try_new(_data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
Ok(Self::from(metadata))
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let opaque = Opaque::new("name", "vendor");
let mut field = Field::new("", DataType::Null, false);
field.try_with_extension_type(opaque.clone())?;
assert_eq!(field.try_extension_type::<Opaque>()?, opaque);
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::Opaque(opaque)
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field = Field::new("", DataType::Null, false).with_metadata(
[(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
r#"{ "type_name": "type", "vendor_name": "vendor" }"#.to_owned(),
)]
.into_iter()
.collect(),
);
field.extension_type::<Opaque>();
}
#[test]
#[should_panic(expected = "Opaque extension types requires metadata")]
fn missing_metadata() {
let field = Field::new("", DataType::Null, false).with_metadata(
[(EXTENSION_TYPE_NAME_KEY.to_owned(), Opaque::NAME.to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<Opaque>();
}
#[test]
#[should_panic(
expected = "Opaque metadata deserialization failed: missing field `vendor_name`"
)]
fn invalid_metadata() {
let field = Field::new("", DataType::Null, false).with_metadata(
[
(EXTENSION_TYPE_NAME_KEY.to_owned(), Opaque::NAME.to_owned()),
(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
r#"{ "type_name": "no-vendor" }"#.to_owned(),
),
]
.into_iter()
.collect(),
);
field.extension_type::<Opaque>();
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/uuid.rs
Vendored
+128
View File
@@ -0,0 +1,128 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! UUID
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid>
use crate::{ArrowError, DataType, extension::ExtensionType};
/// The extension type for `UUID`.
///
/// Extension name: `arrow.uuid`.
///
/// The storage type of the extension is `FixedSizeBinary` with a length of
/// 16 bytes.
///
/// Note:
/// A specific UUID version is not required or guaranteed. This extension
/// represents UUIDs as `FixedSizeBinary(16)` with big-endian notation and
/// does not interpret the bytes in any way.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid>
#[derive(Debug, Default, Clone, Copy, PartialEq)]
pub struct Uuid;
impl ExtensionType for Uuid {
const NAME: &'static str = "arrow.uuid";
type Metadata = ();
fn metadata(&self) -> &Self::Metadata {
&()
}
fn serialize_metadata(&self) -> Option<String> {
None
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
metadata.map_or_else(
|| Ok(()),
|_| {
Err(ArrowError::InvalidArgumentError(
"Uuid extension type expects no metadata".to_owned(),
))
},
)
}
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
match data_type {
DataType::FixedSizeBinary(16) => Ok(()),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"Uuid data type mismatch, expected FixedSizeBinary(16), found {data_type}"
))),
}
}
fn try_new(data_type: &DataType, _metadata: Self::Metadata) -> Result<Self, ArrowError> {
Self.supports_data_type(data_type).map(|_| Self)
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let mut field = Field::new("", DataType::FixedSizeBinary(16), false);
field.try_with_extension_type(Uuid)?;
field.try_extension_type::<Uuid>()?;
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::Uuid(Uuid)
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field = Field::new("", DataType::FixedSizeBinary(16), false);
field.extension_type::<Uuid>();
}
#[test]
#[should_panic(expected = "expected FixedSizeBinary(16), found FixedSizeBinary(8)")]
fn invalid_type() {
Field::new("", DataType::FixedSizeBinary(8), false).with_extension_type(Uuid);
}
#[test]
#[should_panic(expected = "Uuid extension type expects no metadata")]
fn with_metadata() {
let field = Field::new("", DataType::FixedSizeBinary(16), false).with_metadata(
[
(EXTENSION_TYPE_NAME_KEY.to_owned(), Uuid::NAME.to_owned()),
(EXTENSION_TYPE_METADATA_KEY.to_owned(), "".to_owned()),
]
.into_iter()
.collect(),
);
field.extension_type::<Uuid>();
}
}
notes-service/vendor/arrow-schema/src/extension/canonical/variable_shape_tensor.rs
Vendored
+692
View File
@@ -0,0 +1,692 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! VariableShapeTensor
//!
//! <https://arrow.apache.org/docs/format/CanonicalExtensions.html#variable-shape-tensor>
use serde_core::de::{self, MapAccess, Visitor};
use serde_core::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
use crate::{ArrowError, DataType, Field, extension::ExtensionType};
/// The extension type for `VariableShapeTensor`.
///
/// Extension name: `arrow.variable_shape_tensor`.
///
/// The storage type of the extension is: StructArray where struct is composed
/// of data and shape fields describing a single tensor per row:
/// - `data` is a List holding tensor elements (each list element is a single
/// tensor). The List’s value type is the value type of the tensor, such as
/// an integer or floating-point type.
/// - `shape` is a `FixedSizeList<int32>[ndim]` of the tensor shape where the
/// size of the list `ndim` is equal to the number of dimensions of the
/// tensor.
///
/// Extension type parameters:
/// `value_type`: the Arrow data type of individual tensor elements.
///
/// Optional parameters describing the logical layout:
/// - `dim_names`: explicit names to tensor dimensions as an array. The length
/// of it should be equal to the shape length and equal to the number of
/// dimensions.
/// `dim_names` can be used if the dimensions have well-known names and they
/// map to the physical layout (row-major).
/// - `permutation`: indices of the desired ordering of the original
/// dimensions, defined as an array.
/// The indices contain a permutation of the values `[0, 1, .., N-1]` where
/// `N` is the number of dimensions. The permutation indicates which
/// dimension of the logical layout corresponds to which dimension of the
/// physical tensor (the i-th dimension of the logical view corresponds to
/// the dimension with number `permutations[i]` of the physical tensor).
/// Permutation can be useful in case the logical order of the tensor is a
/// permutation of the physical order (row-major).
/// When logical and physical layout are equal, the permutation will always
/// be (`[0, 1, .., N-1]`) and can therefore be left out.
/// - `uniform_shape`: sizes of individual tensor’s dimensions which are
/// guaranteed to stay constant in uniform dimensions and can vary in non-
/// uniform dimensions. This holds over all tensors in the array. Sizes in
/// uniform dimensions are represented with int32 values, while sizes of the
/// non-uniform dimensions are not known in advance and are represented with
/// null. If `uniform_shape` is not provided it is assumed that all
/// dimensions are non-uniform. An array containing a tensor with shape (2,
/// 3, 4) and whose first and last dimensions are uniform would have
/// `uniform_shape` (2, null, 4). This allows for interpreting the tensor
/// correctly without accounting for uniform dimensions while still
/// permitting optional optimizations that take advantage of the uniformity.
///
/// <https://arrow.apache.org/docs/format/CanonicalExtensions.html#variable-shape-tensor>
#[derive(Debug, Clone, PartialEq)]
pub struct VariableShapeTensor {
/// The data type of individual tensor elements.
value_type: DataType,
/// The number of dimensions of the tensor.
dimensions: usize,
/// The metadata of this extension type.
metadata: VariableShapeTensorMetadata,
}
impl VariableShapeTensor {
/// Returns a new variable shape tensor extension type.
///
/// # Error
///
/// Return an error if the provided dimension names, permutations or
/// uniform shapes are invalid.
pub fn try_new(
value_type: DataType,
dimensions: usize,
dimension_names: Option<Vec<String>>,
permutations: Option<Vec<usize>>,
uniform_shapes: Option<Vec<Option<i32>>>,
) -> Result<Self, ArrowError> {
// TODO: are all data types are suitable as value type?
VariableShapeTensorMetadata::try_new(
dimensions,
dimension_names,
permutations,
uniform_shapes,
)
.map(|metadata| Self {
value_type,
dimensions,
metadata,
})
}
/// Returns the value type of the individual tensor elements.
pub fn value_type(&self) -> &DataType {
&self.value_type
}
/// Returns the number of dimensions in this variable shape tensor.
pub fn dimensions(&self) -> usize {
self.dimensions
}
/// Returns the names of the dimensions in this variable shape tensor, if
/// set.
pub fn dimension_names(&self) -> Option<&[String]> {
self.metadata.dimension_names()
}
/// Returns the indices of the desired ordering of the original
/// dimensions, if set.
pub fn permutations(&self) -> Option<&[usize]> {
self.metadata.permutations()
}
/// Returns sizes of individual tensor’s dimensions which are guaranteed
/// to stay constant in uniform dimensions and can vary in non-uniform
/// dimensions.
pub fn uniform_shapes(&self) -> Option<&[Option<i32>]> {
self.metadata.uniform_shapes()
}
}
/// Extension type metadata for [`VariableShapeTensor`].
#[derive(Debug, Clone, PartialEq)]
pub struct VariableShapeTensorMetadata {
/// Explicit names to tensor dimensions.
dim_names: Option<Vec<String>>,
/// Indices of the desired ordering of the original dimensions.
permutations: Option<Vec<usize>>,
/// Sizes of individual tensor's dimensions which are guaranteed to stay
/// constant in uniform dimensions and can vary in non-uniform dimensions.
uniform_shape: Option<Vec<Option<i32>>>,
}
impl Serialize for VariableShapeTensorMetadata {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
use serde_core::ser::SerializeStruct;
let mut state = serializer.serialize_struct("VariableShapeTensorMetadata", 3)?;
state.serialize_field("dim_names", &self.dim_names)?;
state.serialize_field("permutations", &self.permutations)?;
state.serialize_field("uniform_shape", &self.uniform_shape)?;
state.end()
}
}
#[derive(Debug)]
enum MetadataField {
DimNames,
Permutations,
UniformShape,
}
struct MetadataFieldVisitor;
impl<'de> Visitor<'de> for MetadataFieldVisitor {
type Value = MetadataField;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("`dim_names`, `permutations`, or `uniform_shape`")
}
fn visit_str<E>(self, value: &str) -> Result<MetadataField, E>
where
E: de::Error,
{
match value {
"dim_names" => Ok(MetadataField::DimNames),
"permutations" => Ok(MetadataField::Permutations),
"uniform_shape" => Ok(MetadataField::UniformShape),
_ => Err(de::Error::unknown_field(
value,
&["dim_names", "permutations", "uniform_shape"],
)),
}
}
}
impl<'de> Deserialize<'de> for MetadataField {
fn deserialize<D>(deserializer: D) -> Result<MetadataField, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_identifier(MetadataFieldVisitor)
}
}
struct VariableShapeTensorMetadataVisitor;
impl<'de> Visitor<'de> for VariableShapeTensorMetadataVisitor {
type Value = VariableShapeTensorMetadata;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("struct VariableShapeTensorMetadata")
}
fn visit_seq<V>(self, mut seq: V) -> Result<VariableShapeTensorMetadata, V::Error>
where
V: de::SeqAccess<'de>,
{
let dim_names = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(0, &self))?;
let permutations = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(1, &self))?;
let uniform_shape = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(2, &self))?;
Ok(VariableShapeTensorMetadata {
dim_names,
permutations,
uniform_shape,
})
}
fn visit_map<V>(self, mut map: V) -> Result<VariableShapeTensorMetadata, V::Error>
where
V: MapAccess<'de>,
{
let mut dim_names = None;
let mut permutations = None;
let mut uniform_shape = None;
while let Some(key) = map.next_key()? {
match key {
MetadataField::DimNames => {
if dim_names.is_some() {
return Err(de::Error::duplicate_field("dim_names"));
}
dim_names = Some(map.next_value()?);
}
MetadataField::Permutations => {
if permutations.is_some() {
return Err(de::Error::duplicate_field("permutations"));
}
permutations = Some(map.next_value()?);
}
MetadataField::UniformShape => {
if uniform_shape.is_some() {
return Err(de::Error::duplicate_field("uniform_shape"));
}
uniform_shape = Some(map.next_value()?);
}
}
}
Ok(VariableShapeTensorMetadata {
dim_names,
permutations,
uniform_shape,
})
}
}
impl<'de> Deserialize<'de> for VariableShapeTensorMetadata {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_struct(
"VariableShapeTensorMetadata",
&["dim_names", "permutations", "uniform_shape"],
VariableShapeTensorMetadataVisitor,
)
}
}
impl VariableShapeTensorMetadata {
/// Returns metadata for a variable shape tensor extension type.
///
/// # Error
///
/// Return an error if the provided dimension names, permutations or
/// uniform shapes are invalid.
pub fn try_new(
dimensions: usize,
dimension_names: Option<Vec<String>>,
permutations: Option<Vec<usize>>,
uniform_shapes: Option<Vec<Option<i32>>>,
) -> Result<Self, ArrowError> {
let dim_names = dimension_names.map(|dimension_names| {
if dimension_names.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor dimension names size mismatch, expected {dimensions}, found {}", dimension_names.len()
)))
} else {
Ok(dimension_names)
}
}).transpose()?;
let permutations = permutations
.map(|permutations| {
if permutations.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor permutations size mismatch, expected {dimensions}, found {}",
permutations.len()
)))
} else {
let mut sorted_permutations = permutations.clone();
sorted_permutations.sort_unstable();
if (0..dimensions).zip(sorted_permutations).any(|(a, b)| a != b) {
Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: {dimensions}"
)))
} else {
Ok(permutations)
}
}
})
.transpose()?;
let uniform_shape = uniform_shapes
.map(|uniform_shapes| {
if uniform_shapes.len() != dimensions {
Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor uniform shapes size mismatch, expected {dimensions}, found {}",
uniform_shapes.len()
)))
} else {
Ok(uniform_shapes)
}
})
.transpose()?;
Ok(Self {
dim_names,
permutations,
uniform_shape,
})
}
/// Returns the names of the dimensions in this variable shape tensor, if
/// set.
pub fn dimension_names(&self) -> Option<&[String]> {
self.dim_names.as_ref().map(AsRef::as_ref)
}
/// Returns the indices of the desired ordering of the original dimensions,
/// if set.
pub fn permutations(&self) -> Option<&[usize]> {
self.permutations.as_ref().map(AsRef::as_ref)
}
/// Returns sizes of individual tensor’s dimensions which are guaranteed
/// to stay constant in uniform dimensions and can vary in non-uniform
/// dimensions.
pub fn uniform_shapes(&self) -> Option<&[Option<i32>]> {
self.uniform_shape.as_ref().map(AsRef::as_ref)
}
}
impl ExtensionType for VariableShapeTensor {
const NAME: &'static str = "arrow.variable_shape_tensor";
type Metadata = VariableShapeTensorMetadata;
fn metadata(&self) -> &Self::Metadata {
&self.metadata
}
fn serialize_metadata(&self) -> Option<String> {
Some(serde_json::to_string(self.metadata()).expect("metadata serialization"))
}
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
metadata.map_or_else(
|| {
Err(ArrowError::InvalidArgumentError(
"VariableShapeTensor extension types requires metadata".to_owned(),
))
},
|value| {
serde_json::from_str(value).map_err(|e| {
ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor metadata deserialization failed: {e}"
))
})
},
)
}
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
let expected = DataType::Struct(
[
Field::new_list(
"data",
Field::new_list_field(self.value_type.clone(), false),
false,
),
Field::new(
"shape",
DataType::new_fixed_size_list(
DataType::Int32,
i32::try_from(self.dimensions()).expect("overflow"),
false,
),
false,
),
]
.into_iter()
.collect(),
);
data_type
.equals_datatype(&expected)
.then_some(())
.ok_or_else(|| {
ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor data type mismatch, expected {expected}, found {data_type}"
))
})
}
fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
match data_type {
DataType::Struct(fields)
if fields.len() == 2
&& matches!(fields.find("data"), Some((0, _)))
&& matches!(fields.find("shape"), Some((1, _))) =>
{
let shape_field = &fields[1];
match shape_field.data_type() {
DataType::FixedSizeList(_, list_size) => {
let dimensions = usize::try_from(*list_size).expect("conversion failed");
// Make sure the metadata is valid.
let metadata = VariableShapeTensorMetadata::try_new(
dimensions,
metadata.dim_names,
metadata.permutations,
metadata.uniform_shape,
)?;
let data_field = &fields[0];
match data_field.data_type() {
DataType::List(field) => Ok(Self {
value_type: field.data_type().clone(),
dimensions,
metadata,
}),
data_type => Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor data type mismatch, expected List for data field, found {data_type}"
))),
}
}
data_type => Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor data type mismatch, expected FixedSizeList for shape field, found {data_type}"
))),
}
}
data_type => Err(ArrowError::InvalidArgumentError(format!(
"VariableShapeTensor data type mismatch, expected Struct with 2 fields (data and shape), found {data_type}"
))),
}
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "canonical_extension_types")]
use crate::extension::CanonicalExtensionType;
use crate::{
Field,
extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY},
};
use super::*;
#[test]
fn valid() -> Result<(), ArrowError> {
let variable_shape_tensor = VariableShapeTensor::try_new(
DataType::Float32,
3,
Some(vec!["C".to_owned(), "H".to_owned(), "W".to_owned()]),
Some(vec![2, 0, 1]),
Some(vec![Some(400), None, Some(3)]),
)?;
let mut field = Field::new_struct(
"",
vec![
Field::new_list(
"data",
Field::new_list_field(DataType::Float32, false),
false,
),
Field::new_fixed_size_list(
"shape",
Field::new("", DataType::Int32, false),
3,
false,
),
],
false,
);
field.try_with_extension_type(variable_shape_tensor.clone())?;
assert_eq!(
field.try_extension_type::<VariableShapeTensor>()?,
variable_shape_tensor
);
#[cfg(feature = "canonical_extension_types")]
assert_eq!(
field.try_canonical_extension_type()?,
CanonicalExtensionType::VariableShapeTensor(variable_shape_tensor)
);
Ok(())
}
#[test]
#[should_panic(expected = "Field extension type name missing")]
fn missing_name() {
let field = Field::new_struct(
"",
vec![
Field::new_list(
"data",
Field::new_list_field(DataType::Float32, false),
false,
),
Field::new_fixed_size_list(
"shape",
Field::new("", DataType::Int32, false),
3,
false,
),
],
false,
)
.with_metadata(
[(EXTENSION_TYPE_METADATA_KEY.to_owned(), "{}".to_owned())]
.into_iter()
.collect(),
);
field.extension_type::<VariableShapeTensor>();
}
#[test]
#[should_panic(expected = "VariableShapeTensor data type mismatch, expected Struct")]
fn invalid_type() {
let variable_shape_tensor =
VariableShapeTensor::try_new(DataType::Int32, 3, None, None, None).unwrap();
let field = Field::new_struct(
"",
vec![
Field::new_list(
"data",
Field::new_list_field(DataType::Float32, false),
false,
),
Field::new_fixed_size_list(
"shape",
Field::new("", DataType::Int32, false),
3,
false,
),
],
false,
);
field.with_extension_type(variable_shape_tensor);
}
#[test]
#[should_panic(expected = "VariableShapeTensor extension types requires metadata")]
fn missing_metadata() {
let field = Field::new_struct(
"",
vec![
Field::new_list(
"data",
Field::new_list_field(DataType::Float32, false),
false,
),
Field::new_fixed_size_list(
"shape",
Field::new("", DataType::Int32, false),
3,
false,
),
],
false,
)
.with_metadata(
[(
EXTENSION_TYPE_NAME_KEY.to_owned(),
VariableShapeTensor::NAME.to_owned(),
)]
.into_iter()
.collect(),
);
field.extension_type::<VariableShapeTensor>();
}
#[test]
#[should_panic(expected = "VariableShapeTensor metadata deserialization failed: invalid type:")]
fn invalid_metadata() {
let field = Field::new_struct(
"",
vec![
Field::new_list(
"data",
Field::new_list_field(DataType::Float32, false),
false,
),
Field::new_fixed_size_list(
"shape",
Field::new("", DataType::Int32, false),
3,
false,
),
],
false,
)
.with_metadata(
[
(
EXTENSION_TYPE_NAME_KEY.to_owned(),
VariableShapeTensor::NAME.to_owned(),
),
(
EXTENSION_TYPE_METADATA_KEY.to_owned(),
r#"{ "dim_names": [1, null, 3, 4] }"#.to_owned(),
),
]
.into_iter()
.collect(),
);
field.extension_type::<VariableShapeTensor>();
}
#[test]
#[should_panic(
expected = "VariableShapeTensor dimension names size mismatch, expected 3, found 2"
)]
fn invalid_metadata_dimension_names() {
VariableShapeTensor::try_new(
DataType::Float32,
3,
Some(vec!["a".to_owned(), "b".to_owned()]),
None,
None,
)
.unwrap();
}
#[test]
#[should_panic(
expected = "VariableShapeTensor permutations size mismatch, expected 3, found 2"
)]
fn invalid_metadata_permutations_len() {
VariableShapeTensor::try_new(DataType::Float32, 3, None, Some(vec![1, 0]), None).unwrap();
}
#[test]
#[should_panic(
expected = "VariableShapeTensor permutations invalid, expected a permutation of [0, 1, .., N-1], where N is the number of dimensions: 3"
)]
fn invalid_metadata_permutations_values() {
VariableShapeTensor::try_new(DataType::Float32, 3, None, Some(vec![4, 3, 2]), None)
.unwrap();
}
#[test]
#[should_panic(
expected = "VariableShapeTensor uniform shapes size mismatch, expected 3, found 2"
)]
fn invalid_metadata_uniform_shapes() {
VariableShapeTensor::try_new(DataType::Float32, 3, None, None, Some(vec![None, Some(1)]))
.unwrap();
}
}
notes-service/vendor/arrow-schema/src/extension/mod.rs
Vendored
+258
View File
@@ -0,0 +1,258 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Extension types.
#[cfg(feature = "canonical_extension_types")]
mod canonical;
#[cfg(feature = "canonical_extension_types")]
pub use canonical::*;
use crate::{ArrowError, DataType};
/// The metadata key for the string name identifying an [`ExtensionType`].
pub const EXTENSION_TYPE_NAME_KEY: &str = "ARROW:extension:name";
/// The metadata key for a serialized representation of the [`ExtensionType`]
/// necessary to reconstruct the custom type.
pub const EXTENSION_TYPE_METADATA_KEY: &str = "ARROW:extension:metadata";
/// Extension types.
///
/// User-defined “extension” types can be defined setting certain key value
/// pairs in the [`Field`] metadata structure. These extension keys are:
/// - [`EXTENSION_TYPE_NAME_KEY`]
/// - [`EXTENSION_TYPE_METADATA_KEY`]
///
/// Canonical extension types support in this crate requires the
/// `canonical_extension_types` feature.
///
/// Extension types may or may not use the [`EXTENSION_TYPE_METADATA_KEY`]
/// field.
///
/// # Example
///
/// The example below demonstrates how to implement this trait for a `Uuid`
/// type. Note this is not the canonical extension type for `Uuid`, which does
/// not include information about the `Uuid` version.
///
/// ```
/// # use arrow_schema::ArrowError;
/// # fn main() -> Result<(), ArrowError> {
/// use arrow_schema::{DataType, extension::ExtensionType, Field};
/// use std::{fmt, str::FromStr};
///
/// /// The different Uuid versions.
/// #[derive(Clone, Copy, Debug, PartialEq)]
/// enum UuidVersion {
/// V1,
/// V2,
/// V3,
/// V4,
/// V5,
/// V6,
/// V7,
/// V8,
/// }
///
/// // We'll use `Display` to serialize.
/// impl fmt::Display for UuidVersion {
/// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
/// write!(
/// f,
/// "{}",
/// match self {
/// Self::V1 => "V1",
/// Self::V2 => "V2",
/// Self::V3 => "V3",
/// Self::V4 => "V4",
/// Self::V5 => "V5",
/// Self::V6 => "V6",
/// Self::V7 => "V7",
/// Self::V8 => "V8",
/// }
/// )
/// }
/// }
///
/// // And `FromStr` to deserialize.
/// impl FromStr for UuidVersion {
/// type Err = ArrowError;
///
/// fn from_str(s: &str) -> Result<Self, Self::Err> {
/// match s {
/// "V1" => Ok(Self::V1),
/// "V2" => Ok(Self::V2),
/// "V3" => Ok(Self::V3),
/// "V4" => Ok(Self::V4),
/// "V5" => Ok(Self::V5),
/// "V6" => Ok(Self::V6),
/// "V7" => Ok(Self::V7),
/// "V8" => Ok(Self::V8),
/// _ => Err(ArrowError::ParseError("Invalid UuidVersion".to_owned())),
/// }
/// }
/// }
///
/// /// This is the extension type, not the container for Uuid values. It
/// /// stores the Uuid version (this is the metadata of this extension type).
/// #[derive(Clone, Copy, Debug, PartialEq)]
/// struct Uuid(UuidVersion);
///
/// impl ExtensionType for Uuid {
/// // We use a namespace as suggested by the specification.
/// const NAME: &'static str = "myorg.example.uuid";
///
/// // The metadata type is the Uuid version.
/// type Metadata = UuidVersion;
///
/// // We just return a reference to the Uuid version.
/// fn metadata(&self) -> &Self::Metadata {
/// &self.0
/// }
///
/// // We use the `Display` implementation to serialize the Uuid
/// // version.
/// fn serialize_metadata(&self) -> Option<String> {
/// Some(self.0.to_string())
/// }
///
/// // We use the `FromStr` implementation to deserialize the Uuid
/// // version.
/// fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError> {
/// metadata.map_or_else(
/// || {
/// Err(ArrowError::InvalidArgumentError(
/// "Uuid extension type metadata missing".to_owned(),
/// ))
/// },
/// str::parse,
/// )
/// }
///
/// // The only supported data type is `FixedSizeBinary(16)`.
/// fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError> {
/// match data_type {
/// DataType::FixedSizeBinary(16) => Ok(()),
/// data_type => Err(ArrowError::InvalidArgumentError(format!(
/// "Uuid data type mismatch, expected FixedSizeBinary(16), found {data_type}"
/// ))),
/// }
/// }
///
/// // We should always check if the data type is supported before
/// // constructing the extension type.
/// fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError> {
/// let uuid = Self(metadata);
/// uuid.supports_data_type(data_type)?;
/// Ok(uuid)
/// }
/// }
///
/// // We can now construct the extension type.
/// let uuid_v1 = Uuid(UuidVersion::V1);
///
/// // And add it to a field.
/// let mut field =
/// Field::new("", DataType::FixedSizeBinary(16), false).with_extension_type(uuid_v1);
///
/// // And extract it from this field.
/// assert_eq!(field.try_extension_type::<Uuid>()?, uuid_v1);
///
/// // When we try to add this to a field with an unsupported data type we
/// // get an error.
/// let result = Field::new("", DataType::Null, false).try_with_extension_type(uuid_v1);
/// assert!(result.is_err());
/// # Ok(()) }
/// ```
///
/// <https://arrow.apache.org/docs/format/Columnar.html#extension-types>
///
/// [`Field`]: crate::Field
pub trait ExtensionType: Sized {
/// The name identifying this extension type.
///
/// This is the string value that is used for the
/// [`EXTENSION_TYPE_NAME_KEY`] in the [`Field::metadata`] of a [`Field`]
/// to identify this extension type.
///
/// We recommend that you use a “namespace”-style prefix for extension
/// type names to minimize the possibility of conflicts with multiple Arrow
/// readers and writers in the same application. For example, use
/// `myorg.name_of_type` instead of simply `name_of_type`.
///
/// Extension names beginning with `arrow.` are reserved for canonical
/// extension types, they should not be used for third-party extension
/// types.
///
/// Extension names are case-sensitive.
///
/// [`Field`]: crate::Field
/// [`Field::metadata`]: crate::Field::metadata
const NAME: &'static str;
/// The metadata type of this extension type.
///
/// Implementations can use strongly or loosly typed data structures here
/// depending on the complexity of the metadata.
///
/// Implementations can also use `Self` here if the extension type can be
/// constructed directly from its metadata.
///
/// If an extension type defines no metadata it should use `()` to indicate
/// this.
type Metadata;
/// Returns a reference to the metadata of this extension type, or `&()` if
/// if this extension type defines no metadata (`Self::Metadata=()`).
fn metadata(&self) -> &Self::Metadata;
/// Returns the serialized representation of the metadata of this extension
/// type, or `None` if this extension type defines no metadata
/// (`Self::Metadata=()`).
///
/// This is string value that is used for the
/// [`EXTENSION_TYPE_METADATA_KEY`] in the [`Field::metadata`] of a
/// [`Field`].
///
/// [`Field`]: crate::Field
/// [`Field::metadata`]: crate::Field::metadata
fn serialize_metadata(&self) -> Option<String>;
/// Deserialize the metadata of this extension type from the serialized
/// representation of the metadata. An extension type that defines no
/// metadata should expect `None` for the serialized metadata and return
/// `Ok(())`.
///
/// This function should return an error when
/// - expected metadata is missing (for extensions types with non-optional
/// metadata)
/// - unexpected metadata is set (for extension types without metadata)
/// - deserialization of metadata fails
fn deserialize_metadata(metadata: Option<&str>) -> Result<Self::Metadata, ArrowError>;
/// Returns `Ok(())` iff the given data type is supported by this extension
/// type.
fn supports_data_type(&self, data_type: &DataType) -> Result<(), ArrowError>;
/// Construct this extension type for a field with the given data type and
/// metadata.
///
/// This should return an error if the given data type is not supported by
/// this extension type.
fn try_new(data_type: &DataType, metadata: Self::Metadata) -> Result<Self, ArrowError>;
}
notes-service/vendor/arrow-schema/src/ffi.rs
Vendored
+1004
View File
File diff suppressed because it is too large Load Diff
notes-service/vendor/arrow-schema/src/field.rs
Vendored
+1482
View File
File diff suppressed because it is too large Load Diff
notes-service/vendor/arrow-schema/src/fields.rs
Vendored
+1042
View File
File diff suppressed because it is too large Load Diff
notes-service/vendor/arrow-schema/src/lib.rs
Vendored
+217
View File
@@ -0,0 +1,217 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Arrow logical types
#![doc(
html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg",
html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg"
)]
#![cfg_attr(docsrs, feature(doc_cfg))]
#![warn(missing_docs)]
mod datatype;
pub use datatype::*;
use std::fmt::Display;
mod datatype_display;
mod datatype_parse;
mod error;
pub use error::*;
pub mod extension;
mod field;
pub use field::*;
mod fields;
pub use fields::*;
mod schema;
pub use schema::*;
use std::ops;
#[cfg(feature = "ffi")]
pub mod ffi;
/// Options that define the sort order of a given column
///
/// The default sorts equivalently to of `ASC NULLS FIRST` in SQL (i.e.
/// ascending order with nulls sorting before any other values).
///
/// # Example creation
/// ```
/// # use arrow_schema::SortOptions;
/// // configure using explicit initialization
/// let options = SortOptions {
/// descending: false,
/// nulls_first: true,
/// };
/// // Default is ASC NULLs First
/// assert_eq!(options, SortOptions::default());
/// assert_eq!(options.to_string(), "ASC NULLS FIRST");
///
/// // Configure using builder APIs
/// let options = SortOptions::default()
/// .desc()
/// .nulls_first();
/// assert_eq!(options.to_string(), "DESC NULLS FIRST");
///
/// // configure using explicit field values
/// let options = SortOptions::default()
/// .with_descending(false)
/// .with_nulls_first(false);
/// assert_eq!(options.to_string(), "ASC NULLS LAST");
/// ```
///
/// # Example operations
/// It is also possible to negate the sort options using the `!` operator.
/// ```
/// use arrow_schema::SortOptions;
/// let options = !SortOptions::default();
/// assert_eq!(options.to_string(), "DESC NULLS LAST");
/// ```
#[derive(Clone, Hash, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
pub struct SortOptions {
/// Whether to sort in descending order
pub descending: bool,
/// Whether to sort nulls first
pub nulls_first: bool,
}
impl Display for SortOptions {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
if self.descending {
write!(f, "DESC")?;
} else {
write!(f, "ASC")?;
}
if self.nulls_first {
write!(f, " NULLS FIRST")?;
} else {
write!(f, " NULLS LAST")?;
}
Ok(())
}
}
impl SortOptions {
/// Create a new `SortOptions` struct
pub fn new(descending: bool, nulls_first: bool) -> Self {
Self {
descending,
nulls_first,
}
}
/// Set this sort options to sort in descending order
///
/// See [Self::with_descending] to explicitly set the underlying field
pub fn desc(mut self) -> Self {
self.descending = true;
self
}
/// Set this sort options to sort in ascending order
///
/// See [Self::with_descending] to explicitly set the underlying field
pub fn asc(mut self) -> Self {
self.descending = false;
self
}
/// Set this sort options to sort nulls first
///
/// See [Self::with_nulls_first] to explicitly set the underlying field
pub fn nulls_first(mut self) -> Self {
self.nulls_first = true;
self
}
/// Set this sort options to sort nulls last
///
/// See [Self::with_nulls_first] to explicitly set the underlying field
pub fn nulls_last(mut self) -> Self {
self.nulls_first = false;
self
}
/// Set this sort options to sort descending if argument is true
pub fn with_descending(mut self, descending: bool) -> Self {
self.descending = descending;
self
}
/// Set this sort options to sort nulls first if argument is true
pub fn with_nulls_first(mut self, nulls_first: bool) -> Self {
self.nulls_first = nulls_first;
self
}
}
impl Default for SortOptions {
fn default() -> Self {
Self {
descending: false,
// default to nulls first to match spark's behavior
nulls_first: true,
}
}
}
/// `!` operator is overloaded for `SortOptions` to invert boolean
/// fields of the struct.
impl ops::Not for SortOptions {
type Output = SortOptions;
fn not(self) -> SortOptions {
SortOptions {
descending: !self.descending,
nulls_first: !self.nulls_first,
}
}
}
#[test]
fn test_overloaded_not_sort_options() {
let sort_options_array = [
SortOptions {
descending: false,
nulls_first: false,
},
SortOptions {
descending: false,
nulls_first: true,
},
SortOptions {
descending: true,
nulls_first: false,
},
SortOptions {
descending: true,
nulls_first: true,
},
];
assert!((!sort_options_array[0]).descending);
assert!((!sort_options_array[0]).nulls_first);
assert!((!sort_options_array[1]).descending);
assert!(!(!sort_options_array[1]).nulls_first);
assert!(!(!sort_options_array[2]).descending);
assert!((!sort_options_array[2]).nulls_first);
assert!(!(!sort_options_array[3]).descending);
assert!(!(!sort_options_array[3]).nulls_first);
}
notes-service/vendor/arrow-schema/src/schema.rs
Vendored
+1505
View File
File diff suppressed because it is too large Load Diff