feat: enhanced cargo workspace, NM usage, Dockerfile

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+name: Publish CKAN-Geoconnex JSONL file as latest release
+on:
+  release:
+    types: [published]
+permissions:
+  # To upload to releases
+  contents: write
+jobs:
+  publish:
+    name: Publish JSONL file
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install Rust stable toolchain
+        uses: dtolnay/rust-toolchain@stable
+      - name: Cache cargo deps and target folder
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: ". -> target"
+      - name: Run generate_release crate and upload JSONL file
+        run: |
+          cd ${{github.workspace}}
+          cargo run -p generate_release --release --verbose > ckan-geoconnex-web-resources.jsonl
+          gh release upload ${{github.event.release.tag_name}} ckan-geoconnex-web-resources.jsonl
+        env:
+          GITHUB_TOKEN: ${{ github.TOKEN }}

Cargo.lock

@@ -45,9 +45,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
 
 [[package]]
 name = "autocfg"
-version = "1.5.0"
+version = "1.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-lc-rs"
@@ -94,15 +94,15 @@ checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
 
 [[package]]
 name = "bitflags"
-version = "2.11.1"
+version = "2.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
+checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8"
 
 [[package]]
 name = "bon"
-version = "3.9.1"
+version = "3.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
+checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561"
 dependencies = [
  "bon-macros",
  "rustversion",
@@ -110,9 +110,9 @@ dependencies = [
 
 [[package]]
 name = "bon-macros"
-version = "3.9.1"
+version = "3.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
+checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f"
 dependencies = [
  "darling",
  "ident_case",
@@ -129,11 +129,20 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c"
 
+[[package]]
+name = "bulk_loader"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "reqwest 0.13.4",
+ "tokio",
+]
+
 [[package]]
 name = "bumpalo"
-version = "3.20.2"
+version = "3.20.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
 
 [[package]]
 name = "bytecount"
@@ -149,9 +158,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"
-version = "1.2.62"
+version = "1.2.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
+checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -166,30 +175,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
 
 [[package]]
-name = "ckan_geoconnex_bulk_runner"
+name = "cfg_aliases"
-version = "0.1.0"
+version = "0.2.1"
-dependencies = [
+source = "registry+https://github.com/rust-lang/crates.io-index"
- "anyhow",
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
- "ckanaction",
- "jsonschema",
- "serde_json",
- "tokio",
-]
 
 [[package]]
 name = "ckan_geoconnex_bulk_runner_py"
 version = "0.1.0"
 dependencies = [
- "ckan_geoconnex_bulk_runner",
+ "geoconnex_utils",
  "pyo3",
  "serde_json",
 ]
 
 [[package]]
 name = "ckanaction"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2e5332c456e22de38ddf1c8ed0cdfaeba4ab432f5de8cd971b6450ef62f8add"
+checksum = "4fe1081a4952a4d708e7583a8151588db759aa4d9631198fc8c9ead6b0f9a358"
 dependencies = [
  "bon",
  "reqwest 0.12.28",
@@ -285,9 +289,9 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
 
 [[package]]
 name = "displaydoc"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -487,6 +491,29 @@ dependencies = [
  "slab",
 ]
 
+[[package]]
+name = "generate_release"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "ckanaction",
+ "geoconnex_utils",
+ "jsonschema",
+ "serde_json",
+ "tokio",
+]
+
+[[package]]
+name = "geoconnex_utils"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "ckanaction",
+ "jsonschema",
+ "serde_json",
+ "tokio",
+]
+
 [[package]]
 name = "getrandom"
 version = "0.2.17"
@@ -494,8 +521,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "wasi",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -527,9 +556,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.14"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -578,9 +607,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "http"
-version = "1.4.0"
+version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425"
 dependencies = [
  "bytes",
  "itoa",
@@ -617,9 +646,9 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
 
 [[package]]
 name = "hyper"
-version = "1.9.0"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -892,13 +921,12 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.98"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
 dependencies = [
  "cfg-if",
  "futures-util",
- "once_cell",
  "wasm-bindgen",
 ]
 
@@ -923,7 +951,7 @@ dependencies = [
  "referencing",
  "regex",
  "regex-syntax",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
  "rustls",
  "serde",
  "serde_json",
@@ -972,15 +1000,21 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.29"
+version = "0.4.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a"
+
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
 
 [[package]]
 name = "memchr"
-version = "2.8.0"
+version = "2.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4"
 
 [[package]]
 name = "micromap"
@@ -1006,9 +1040,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.2.0"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
 dependencies = [
  "libc",
  "wasi",
@@ -1119,9 +1153,9 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "openssl"
-version = "0.10.80"
+version = "0.10.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -1150,9 +1184,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.116"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
 dependencies = [
  "cc",
  "libc",
@@ -1222,6 +1256,15 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
 [[package]]
 name = "prettyplease"
 version = "0.2.37"
@@ -1299,6 +1342,62 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "quinn"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
+dependencies = [
+ "bytes",
+ "cfg_aliases",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+dependencies = [
+ "aws-lc-rs",
+ "bytes",
+ "getrandom 0.3.4",
+ "lru-slab",
+ "rand",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "quote"
 version = "1.0.45"
@@ -1320,6 +1419,35 @@ version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
 
+[[package]]
+name = "rand"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
+dependencies = [
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.9.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
+dependencies = [
+ "getrandom 0.3.4",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -1368,9 +1496,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
+checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -1391,9 +1519,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.10"
+version = "0.8.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
+checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
 
 [[package]]
 name = "reqwest"
@@ -1441,12 +1569,13 @@ dependencies = [
 
 [[package]]
 name = "reqwest"
-version = "0.13.3"
+version = "0.13.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
 dependencies = [
  "base64",
  "bytes",
+ "encoding_rs",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1459,8 +1588,10 @@ dependencies = [
  "hyper-util",
  "js-sys",
  "log",
+ "mime",
  "percent-encoding",
  "pin-project-lite",
+ "quinn",
  "rustls",
  "rustls-pki-types",
  "rustls-platform-verifier",
@@ -1492,6 +1623,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.1"
@@ -1530,9 +1667,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
 dependencies = [
  "openssl-probe",
  "rustls-pki-types",
@@ -1546,6 +1683,7 @@ version = "1.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
 dependencies = [
+ "web-time",
  "zeroize",
 ]
 
@@ -1685,9 +1823,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.149"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
 dependencies = [
  "itoa",
  "memchr",
@@ -1710,9 +1848,9 @@ dependencies = [
 
 [[package]]
 name = "shlex"
-version = "1.3.0"
+version = "2.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
 
 [[package]]
 name = "signal-hook-registry"
@@ -1748,15 +1886,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
 
 [[package]]
 name = "smallvec"
-version = "1.15.1"
+version = "1.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90"
 
 [[package]]
 name = "socket2"
-version = "0.6.3"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
+checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
 dependencies = [
  "libc",
  "windows-sys 0.61.2",
@@ -1881,6 +2019,21 @@ dependencies = [
  "zerovec",
 ]
 
+[[package]]
+name = "tinyvec"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
 [[package]]
 name = "tokio"
 version = "1.52.3"
@@ -2115,9 +2268,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
  "wit-bindgen 0.57.1",
 ]
@@ -2133,9 +2286,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.121"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -2146,9 +2299,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.71"
+version = "0.4.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -2156,9 +2309,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.121"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2166,9 +2319,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.121"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -2179,9 +2332,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.121"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
 dependencies = [
  "unicode-ident",
 ]
@@ -2235,9 +2388,19 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.98"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -2480,9 +2643,9 @@ checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
 
 [[package]]
 name = "yoke"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
+checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
 dependencies = [
  "stable_deref_trait",
  "yoke-derive",
@@ -2503,18 +2666,18 @@ dependencies = [
 
 [[package]]
 name = "zerocopy"
-version = "0.8.48"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
+checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f"
 dependencies = [
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.8.48"
+version = "0.8.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
+checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2544,9 +2707,9 @@ dependencies = [
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"

Cargo.toml

@@ -1,6 +1,8 @@
 [workspace]
 resolver = "3"
 members = [
-  "ckan_geoconnex_bulk_runner",
+  "bulk_loader",
   "ckan_geoconnex_bulk_runner_py",
+  "generate_release",
+  "geoconnex_utils",
 ]

README.md

@@ -4,38 +4,11 @@ https://github.com/user-attachments/assets/779fe866-d511-44f3-91a9-a1c2e1cfa189
 
 > Status: This codebase is currently a work in progress and more documentation is planned.
 
-The `ckan_geoconnex_bulk_runner` codebase is meant to run as a container for a bulk integration of a [CKAN](https://ckan.org) instance's relevant datasets and vector geospatial features (e.g. for water data hubs) to the [Geoconnex](https://geoconnex.us) knowledge graph. The codebase ultimately runs as a program outputting to standard output JSON-LD on a new line for each approved dataset/location which the Geoconnex crawler then uses to update the Geoconnex knowledge graph.
+The `ckan_geoconnex_bulk_runner` codebase is part of a multi-service infrastructure to sync water data hubs using [CKAN](https://ckan.org) to the [Geoconnex](https://geoconnex.us) knowledge graph.
 
-Refer to the "Contributing via Bulk Containers" documentation here for more information: https://docs.geoconnex.us/contributing/bulk/
+- [**geoconnex_utils**](geoconnex_utils): Helper functions used throughout the ckan_geoconnex_bulk_runner project including JSON-LD construction and JSON schema validation.
+- [**geoconnex_release**](geoconnex_release): Compatible CKAN datasets and vector geospatial features for all connected water data hubs are uploaded to a `ckan-geoconnex-web-resources.jsonl` file in the latest GitHub release.
+- [**bulk_loader**](bulk_loader): Requests and outputs the latest JSONL file from the latest GitHub release. This is ran as a Docker container by Geoconnex on a periodic frequency to upload all water data hub web resources to the Geoconnex knowledge graph following the Geoconnex [bulk contribution specification](https://docs.geoconnex.us/contributing/bulk/).
+- [**ckan_geoconnex_bulk_runner_py](ckan_geoconnex_bulk_runner_py): Python library intended for usage by the ckanext-gztr and [DataPusher+](https://github.com/dathere/datapusher-plus) CKAN extensions.
 
-This runner is expected to be implemented for a water data hub with the relevant fields and/or ckanext-gztr (not open-source yet) and/or [DataPusher+](https://github.com/dathere/datapusher-plus) enabled. For questions reach out to [datHere](https://dathere.com), [Center for Geospatial Solutions](https://cgsearth.org/), or add an issue/discussion.
+This runner is expected to be implemented for a water data hub with the relevant fields and/or ckanext-gztr (not open-source yet) and/or DataPusher+ enabled. For questions reach out to [datHere](https://dathere.com), [Center for Geospatial Solutions](https://cgsearth.org/), or add an issue/discussion.
-
-## Installation and setup
-
-```bash
-cargo run -p ckan_geoconnex_bulk_runner --release
-```
-
-To ignore standard error output and only show valid output:
-
-```bash
-cargo run -p ckan_geoconnex_bulk_runner --release  2>/dev/null
-```
-
-## Run tests
-
-```bash
-cargo test -p ckan_geoconnex_bulk_runner
-```
-
-To include print statements in test output, run:
-
-```bash
-cargo test -p ckan_geoconnex_bulk_runner -- --nocapture
-```
-
-If you have the local dump files setup available you can run those tests with:
-
-```bash
-cargo test -p ckan_geoconnex_bulk_runner -F local -- --nocapture
-```

bulk_loader/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "bulk_loader"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.102"
+reqwest = "0.13.4"
+tokio = { version = "1.52.1", features = ["full"] }

bulk_loader/Dockerfile

@@ -0,0 +1,9 @@
+FROM rust:1.96 AS builder
+WORKDIR /app
+RUN rustup set profile minimal
+COPY . .
+RUN cargo build --release
+
+FROM ubuntu:latest
+COPY --from=builder /app/target/release/bulk_loader /
+ENTRYPOINT ["/bulk_loader"]

bulk_loader/src/main.rs

@@ -0,0 +1,17 @@
+use anyhow::Result;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    // Get latest release data which is organized as a single JSONL file
+    // at https://github.com/dathere/ckan_geoconnex_bulk_runner/releases/latest
+    let body = reqwest::get("https://github.com/dathere/ckan_geoconnex_bulk_runner/releases/latest/download/ckan-geoconnex-web-resources.jsonl")
+        .await?
+        .text()
+        .await?;
+    // Print each line to stdout
+    for line in body.lines() {
+        println!("{line}");
+    }
+
+    Ok(())
+}

ckan_geoconnex_bulk_runner/config/example.toml

@@ -1,9 +0,0 @@
-# Example configuration file. Customize for a specific CKAN instance when making a new Docker image.
-
-# This is the path after https://geoconnex.us/ and before /{id}.
-# Same as namespace location at https://github.com/internetofwater/geoconnex.us.
-namespace_path = "nmwdh/ckan-dataset"
-# The base URL for the CKAN instance to run API requests to.
-ckan_url = "http://localhost:5000"
-# The name of the key in /package_show for a given CKAN dataset where the Geoconnex `about` field is populated.
-about_field_name = "geoconnex_about"

ckan_geoconnex_bulk_runner_py/Cargo.toml

@@ -9,6 +9,6 @@ crate-type = ["cdylib"]
 name = "ckan_geoconnex_bulk_runner_py"
 
 [dependencies]
-ckan_geoconnex_bulk_runner = { path = "../ckan_geoconnex_bulk_runner" }
+geoconnex_utils = { path = "../geoconnex_utils" }
 pyo3 = "0.28.3"
 serde_json = "1.0.149"

ckan_geoconnex_bulk_runner_py/src/lib.rs

@@ -14,9 +14,8 @@ mod ckan_geoconnex_bulk_runner_py {
     fn construct_dataset_jsonld_from_metadata(dataset_metadata: String) -> PyResult<String> {
         match serde_json::to_value(dataset_metadata) {
             Ok(dataset_json) => {
-                match ckan_geoconnex_bulk_runner::jsonld::construct_dataset_jsonld_from_metadata(
+                match geoconnex_utils::jsonld::construct_dataset_jsonld_from_metadata(dataset_json)
-                    dataset_json,
+                {
-                ) {
                     Ok(jsonld) => serde_json::to_string(&jsonld).map_err(|e| {
                         PyException::new_err(format!(
                             "Error when converting JSON-LD to string: {e}"

generate_release/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "generate_release"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1.0.102"
+ckanaction = "0.2.1"
+geoconnex_utils = { path = "../geoconnex_utils" }
+jsonschema = "0.46.4"
+serde_json = "1.0.149"
+tokio = { version = "1.52.1", features = ["full"] }
+
+[features]
+local = []

generate_release/src/main.rs

@@ -1,14 +1,19 @@
 use anyhow::{Result, bail};
-use ckan_geoconnex_bulk_runner::{
+use geoconnex_utils::{jsonld::construct_dataset_jsonld_from_metadata, schema::get_dataset_schema};
-    jsonld::construct_dataset_jsonld_from_metadata, schema::get_dataset_schema,
+use std::collections::HashMap;
-};
-
-// TODO: Ensure error output is only streamed to stderr as per Geoconnex docs
 
 #[tokio::main]
 async fn main() -> Result<()> {
+    // Identify required header data
+    let Ok(nmwdc_token) = std::env::var("NMWDC_API_BULK_LOADER_TOKEN") else {
+        bail!("Could not find environment variable NMWDC_API_BULK_LOADER_TOKEN.");
+    };
+    let mut headers = HashMap::new();
+    headers.insert("x-geoconnex-runner".to_string(), nmwdc_token);
+
     let ckan = ckanaction::CKAN::builder()
-        .url("http://localhost:5000")
+        .url("https://catalog.newmexicowaterdata.org")
+        .headers(headers)
         .build();
 
     // Paginate through /api/3/action/package_list until only an empty array is returned
@@ -65,9 +70,12 @@ async fn main() -> Result<()> {
                             dataset_metadata.to_owned(),
                         ) {
                             Ok(j) => j,
-                            Err(e) => bail!(
+                            Err(e) => {
-                                "Error while attempting to construct JSON-LD from dataset's metadata: {e}"
+                                eprintln!(
-                            ),
+                                    "Error while attempting to construct JSON-LD from dataset's metadata: {e}"
+                                );
+                                continue;
+                            }
                         };
                         // 3. Validate the JSON-LD against the dataset JSON schema
                         if jsonschema::validate(&get_dataset_schema(), &jsonld).is_ok() {
@@ -75,7 +83,7 @@ async fn main() -> Result<()> {
                             println!("{jsonld}");
                         } else {
                             eprintln!("JSON-LD for {dataset_name} is not valid.");
-                            eprintln!("{jsonld}");
+                            // eprintln!("{jsonld}");
                         }
                     } else {
                         bail!(

geoconnex_utils/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
-name = "ckan_geoconnex_bulk_runner"
+name = "geoconnex_utils"
 version = "0.1.0"
 edition = "2024"
 
 [dependencies]
 anyhow = "1.0.102"
-ckanaction = "0.2.0"
+ckanaction = "0.2.1"
 jsonschema = "0.46.4"
 serde_json = "1.0.149"
 tokio = { version = "1.52.1", features = ["full"] }

geoconnex_utils/src/jsonld.rs

@@ -5,6 +5,7 @@ pub fn construct_dataset_jsonld_from_metadata(
     dataset_metadata: serde_json::Value,
 ) -> Result<serde_json::Value> {
     let dataset_id = dataset_metadata.get("id").unwrap().as_str().unwrap();
+    eprintln!("Attempting to construct JSON-LD for dataset {dataset_id}");
     let dataset_title = dataset_metadata.get("title").unwrap().as_str().unwrap();
     let organization_name = dataset_metadata
         .get("organization")
@@ -31,7 +32,7 @@ pub fn construct_dataset_jsonld_from_metadata(
             };
             let Some(features) = features_value.as_array() else {
                 bail!(
-                    "Eerror while attempting to take features value as array from spatial_full GeoJSON."
+                    "Error while attempting to take features value as array from spatial_full GeoJSON."
                 );
             };
             for feature in features {
@@ -59,14 +60,14 @@ pub fn construct_dataset_jsonld_from_metadata(
         },
         "@type": "Dataset",
         // TODO: Customize namespace based on CKAN instance being used
-        "@id": format!("https://geoconnex.us/nmwdh/ckan-datasets/{dataset_id}"),
+        "@id": format!("https://geoconnex.us/ckan/sandbox/{dataset_id}"),
         "name": dataset_title,
         "provider": {
             "@type": "Organization",
             "name": organization_name
         },
         // TODO: Customize CKAN instance URL based on CKAN instance being used
-        "url": format!("http://localhost:5000/dataset/{dataset_id}")
+        "url": format!("https://sandbox.opendataportal.us/dataset/{dataset_id}")
     });
     let jsonld_map = jsonld.as_object_mut().unwrap();
     if about.len() > 0 {

geoconnex_utils/tests/validate_jsonld.rs

@@ -1,7 +1,7 @@
-use anyhow::{Result, bail};
+use anyhow::{bail, Result};
 #[cfg(feature = "local")]
-use ckan_geoconnex_bulk_runner::schema::get_dataset_schema;
+use geoconnex_utils::schema::get_dataset_schema;
-use ckan_geoconnex_bulk_runner::schema::get_location_schema;
+use geoconnex_utils::schema::get_location_schema;
 use serde_json::json;
 #[cfg(feature = "local")]
 use std::{