diff --git a/apps/cli/executables/pexable/productfetcher/poetry.lock b/apps/cli/executables/pexable/productfetcher/poetry.lock
index de43cf667331a1a2cdf00d0299fc942bf602dcb0..8c0102ab048dfd12ee61bc72a5e77c451fb65564 100644
--- a/apps/cli/executables/pexable/productfetcher/poetry.lock
+++ b/apps/cli/executables/pexable/productfetcher/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
[[package]]
name = "beautifulsoup4"
@@ -139,6 +139,109 @@ files = [
{file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
]
+[[package]]
+name = "crc32c"
+version = "2.4"
+description = "A python package implementing the crc32c algorithm in hardware and software"
+optional = false
+python-versions = "*"
+files = [
+ {file = "crc32c-2.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b26976cfd4ed65b8ea91e5b5dcff27c4bf227594216d89944cf04e3821b673a3"},
+ {file = "crc32c-2.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:95d46af714e01a091b17ac60526c7d78f21bf0c390cf013efb918a56191d0782"},
+ {file = "crc32c-2.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4e01e3651a7d56808b792a381485e13366fd20a507bf41e48f266ca17af4c068"},
+ {file = "crc32c-2.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d73e77f63f1a9fc3a6497b539fbcfde62dfe90daa2ad6edaad7df595164f76c0"},
+ {file = "crc32c-2.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95281a7e1192c4a046986e8e3e4cf766dbd45e91b22b501b1ad2b19446e5a825"},
+ {file = "crc32c-2.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4c9f501bb48e0a24bef9f0965ef83fcf882dfc2d4e9187e2721ba40d1e75c0e"},
+ {file = "crc32c-2.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b55f015d76b220eb4cb213ab3353f17c96fb76ed3d06c6cf404c091705302d89"},
+ {file = "crc32c-2.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:05f958a1a3f1a60c86f0f60079ec57b626251331b321a3b261c2a3ee29026d5c"},
+ {file = "crc32c-2.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3c12a2c03fa27f5cf4d28f59c6923568351e0924e2996d878dbeacf83faea4d6"},
+ {file = "crc32c-2.4-cp310-cp310-win32.whl", hash = "sha256:2a64596f336c9d57fed222bac0b6b37432c9269d86624b7a3dfb9ba018e6c543"},
+ {file = "crc32c-2.4-cp310-cp310-win_amd64.whl", hash = "sha256:c4f3f547f68715525d8fd0bb04c890e4968c6c20d04994dbac533631f7c4ad10"},
+ {file = "crc32c-2.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:77c4066f2093f50f2c400ac50b88b07739747088d1498c00d24a3166caed7213"},
+ {file = "crc32c-2.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:37a8adef2aecc4158b2e3cd60e65b89a67368b3d01cbf05a80a57dd3a8cb785f"},
+ {file = "crc32c-2.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9bf83f8840d256d2d05d83c7ca2f5971e8a022e6106a14a70ea3667aa5981463"},
+ {file = "crc32c-2.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a76ead3fbf8aea7170c5c32b680491551a6195f17b0e05765fb4b452594178ea"},
+ {file = "crc32c-2.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:28c335908504bc84a5779cc41a9200178562c2c79050d8458ebe94d5702a8cf0"},
+ {file = "crc32c-2.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4262e7bcb578f4e86eb04c4bea02762889249480e0154adaed01bec4c25a068"},
+ {file = "crc32c-2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:83e79881d47a0773137926f5beec89575906f3dd057e6b4a181c890840b9e08d"},
+ {file = "crc32c-2.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f50a52c10682e56a8fe6934a4e86600e93c4b42a5481b8ff3da0fecca1fa7bad"},
+ {file = "crc32c-2.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db91b486df07a622553b64fb5d2a67696240914c63d7230410da3a636ea859dc"},
+ {file = "crc32c-2.4-cp311-cp311-win32.whl", hash = "sha256:988c608cfc8e708690102e7abb5e2f98e68641cf0ab43ef8777430a963b02d70"},
+ {file = "crc32c-2.4-cp311-cp311-win_amd64.whl", hash = "sha256:977a748205dc7ab775b26a4f1ba5372b0ddf3e9bcb24ec78fcd8e9eaf836e572"},
+ {file = "crc32c-2.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1f7a2de234982a4a5b299b753f08c41257912c5d12076558238ceb4efdc5b07b"},
+ {file = "crc32c-2.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a95845816b739860684bac663bc98cc1a4358711a73d8602b4efa3611cab4d06"},
+ {file = "crc32c-2.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc34af7d14ae61061d0f15eb3b819692471c5e7403fa2d9284c5c6fdbe621c9b"},
+ {file = "crc32c-2.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fca7abb3cea2453e136b2439f5bc9202660d057835b7005b2a589c2a6daa4c0"},
+ {file = "crc32c-2.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f529dbc0e22a1dbe86de712367960374073a2dca543a18c2b2e5ea77fda231b5"},
+ {file = "crc32c-2.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aec0399578bc380497dc9441d58c7368d83c89f38423f799a315be08b3f0957b"},
+ {file = "crc32c-2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a66e68e9cc4c05db725235079d1f3f647120622e29761cc1e8c81b55eb7810dc"},
+ {file = "crc32c-2.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:15cc1e2d7749b3096cbca08a707fafb61d63a3d5306aa42ce7a131d504d3b931"},
+ {file = "crc32c-2.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3975c76e10386e5a9049bc948c13c5cb996e26e05b736723c4c88f4f034c1776"},
+ {file = "crc32c-2.4-cp312-cp312-win32.whl", hash = "sha256:8167d620bf80ecd0eafc0b0dd9b039e6ec181eed06288ad6cec346bc058dc1d2"},
+ {file = "crc32c-2.4-cp312-cp312-win_amd64.whl", hash = "sha256:d39e4af6d69cb23fa86030dc50737abd6d5f399dc95ab1bd8a89d69329da8aeb"},
+ {file = "crc32c-2.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:3742a602867966b088a6ba245fa44f5f6c232e352e2474bf8a53992df97f3f9b"},
+ {file = "crc32c-2.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c360a88b6acfe0067516df616e7a9c8174c99803ef39815a8bb57efae93b72e"},
+ {file = "crc32c-2.4-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c58243f207c836db732165fbe3cf113029476e1e8f58f3c7d0b7fa29c99dcec8"},
+ {file = "crc32c-2.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3c10c238dd16747a532496ef608ceff755c2635c21657248a8c032d32c8e70"},
+ {file = "crc32c-2.4-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c9cde473e48e566f4733607ab91e25ff13fd6859bb95fb1bef5ad8bb6e182f15"},
+ {file = "crc32c-2.4-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f23eaa542d2ebb0222d96d217a346c398f0ffb1535c694a76a3d71a2752eadad"},
+ {file = "crc32c-2.4-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fc8769c407f98a659c7694399d0b8773f0f7b9b5424e09e45d4e1036037efdf7"},
+ {file = "crc32c-2.4-cp36-cp36m-win32.whl", hash = "sha256:34ea2c204b8986c31789e8ff297bbd88e340a8a8c53d02d1802088494322911d"},
+ {file = "crc32c-2.4-cp36-cp36m-win_amd64.whl", hash = "sha256:5c291a08519d170cd58787fe6d70e3828df10a86c72cefd2758dd6434e05888f"},
+ {file = "crc32c-2.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:031353d47819b0d6d4ea683dd5b6aecc72e3682449e88457f38a670cb3f354b8"},
+ {file = "crc32c-2.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce14694790a85b60cf3f4a3244f6bb6cf2f303e51538734a23f9a3698ba7d53b"},
+ {file = "crc32c-2.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:954eff112ad1586c3b719c3d6e5c00aa97c9484fdb629643b0b800a9416daa8d"},
+ {file = "crc32c-2.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bc180735766f8d85142ee89807c3414b0179456ca5149018d697ead17c3c014"},
+ {file = "crc32c-2.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:96c3e4c940ff987cb15f41e0f5bc582c0dfa9c961f82686ee8478afb38056c13"},
+ {file = "crc32c-2.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1d5df23c0ef1564ea5ffbb6476f34e0f289fc18c48562e13fc7a402ba02847b7"},
+ {file = "crc32c-2.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:057d1d22033e44611eb69ffbb67bcaee6cc3dab3e1635874908de959aece279f"},
+ {file = "crc32c-2.4-cp37-cp37m-win32.whl", hash = "sha256:866747ffbcd86b49176ddcc9234ef2ddea58d77f3fa9fa1207633fcbdfae37bd"},
+ {file = "crc32c-2.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6464b53971ac49098fec692cf022d69548fbd07a02290033f8ed00a23d69055d"},
+ {file = "crc32c-2.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b9ec8a8c91cbd17dd86bd75ab96db6adf4fcb450e7f0cbed6140174251ad79a3"},
+ {file = "crc32c-2.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:10817aa19b6c97d358a28f49ff85057638ae6e750d602b363b138a1049299e97"},
+ {file = "crc32c-2.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:16b2fad68d0f79cd4203940988a3a4bd406a283e73d9f77a03ad8ee6ca92ae20"},
+ {file = "crc32c-2.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fbe7a12a1ca4de17490f08518cff6e2a83610fa8c38c1cf7eadbcb7b7aad806"},
+ {file = "crc32c-2.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:031339d84e80d8724e27f208bf4899ffc87854e0e6b743fc98f7d36162d7ce3c"},
+ {file = "crc32c-2.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2e5dce1731925a9395240cd33edf15aaf17521694acea5596f5a21868492169"},
+ {file = "crc32c-2.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc590453e1f3fa940c845338419947ffd765860f6f421abeb20bfe4d86517b5a"},
+ {file = "crc32c-2.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9d85331942b9a9af00648cde7ee03fb6b51b7cf49cf98384c642b453874e30f5"},
+ {file = "crc32c-2.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1ccc74911a9815f0ec1d21130aa94b4d33ce04b4612e0144621dd5a15708ef5d"},
+ {file = "crc32c-2.4-cp38-cp38-win32.whl", hash = "sha256:8bed35233cfe0af313b3b881af16f3ea9d4fce92e7c3c17120011103e063374a"},
+ {file = "crc32c-2.4-cp38-cp38-win_amd64.whl", hash = "sha256:5768230e26ebfa35e167eb58f7d36e72d290545e436cf3e50ec94071743be16b"},
+ {file = "crc32c-2.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7020406b9732001f29f4d77fbec664b3f85c1fcf5f459ae3977fa6e4846a23b6"},
+ {file = "crc32c-2.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8549d8405eae1b420ecba6d32ca8d8e37de98722b18f4254bc60fff7162b56d9"},
+ {file = "crc32c-2.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e19ff2bb106a2c1afed9ffa1c38c877d0003cd5b4e8d821afcff743e18c8777"},
+ {file = "crc32c-2.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc06d46ae9390ef2ef973f1e609f388c40b2d120aa81bccc506a48534d7ee9b5"},
+ {file = "crc32c-2.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e9ee7970e7abb18f4a3700c0a0e10e662feb4a20cc4b7a5cb69693dc9d4cb858"},
+ {file = "crc32c-2.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5124e149fdb0b6e1c58024be3b944e3650125d209ef3a2b07de303b6d619c60e"},
+ {file = "crc32c-2.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9b9b218cee7d820512a5d76d1e0d3aeeb24886f5a9f543685f5acb23d956c3f1"},
+ {file = "crc32c-2.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:94820b8e069d5eb711afed7920b2b9eb7919869204e09434d71c7d1b8e1a00bf"},
+ {file = "crc32c-2.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a176337bdde989f3a1f0a930619baf2406521c12be744074c6f0eb4e7b6ba346"},
+ {file = "crc32c-2.4-cp39-cp39-win32.whl", hash = "sha256:240f60443c1b8196b619e899c4eae30a09e38151f92c4436d24e119647e9f71a"},
+ {file = "crc32c-2.4-cp39-cp39-win_amd64.whl", hash = "sha256:55775065e2c806acc3ecd1e72de8936e1c56dd967d81c89bc219477d65333c01"},
+ {file = "crc32c-2.4-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:9247cc523fbb7e2166a7058f6723c444fd07b9bf5c87c5c242fd635d46fe9089"},
+ {file = "crc32c-2.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a93e778a60cb8bb609dc4d596ccedb933e833c94d828820bccb929829321dbc0"},
+ {file = "crc32c-2.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:625c9fafb4ed85a335ca95c2869c0ad873ae8f18aa8a149314a3b1129b36cda7"},
+ {file = "crc32c-2.4-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96a8c2d32dc07872c16113bf2a51803d5101ff7e616d9cd94fad9e24066e75cb"},
+ {file = "crc32c-2.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5a0f7eeaa37cff2ccfaa155906e7079716535b02c6211bf1be58542a3524558d"},
+ {file = "crc32c-2.4-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a00edfa728279367de8b394822b3439fa5927002b53d98445b6ec628f4269b0f"},
+ {file = "crc32c-2.4-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cbbefad3c6aea43e74c5a6e968b4226426ba01b88a5922778566bdba65c3f74"},
+ {file = "crc32c-2.4-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39aedd018ec2e70f7322778a8a2b7ed1a13b183aa6b9622e0f8df59e6af2f5bf"},
+ {file = "crc32c-2.4-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0429062fe7d6568d44548dccdbd0a8a63fc7b0fdc20bbde91f5b1e3ea3cccab5"},
+ {file = "crc32c-2.4-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:fd082fa802d2183179554ad6a202c3ec7c101b2eda283e9074e0f7b9a68f5504"},
+ {file = "crc32c-2.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:62f4cd80d02b0337b33ddd3889267f2ad93472ea36ae81656ad50340aa8e5784"},
+ {file = "crc32c-2.4-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c0fe3b69f6c1ec9f0600be6564439da7e2af8ee9ffd5ac7b213591671e09a66"},
+ {file = "crc32c-2.4-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a90b7c3305ec5d03c5791f77ba7a3cb49ac74de762e2307c62dd6f3985b27a9d"},
+ {file = "crc32c-2.4-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:512af034f21f0fe4f0b599cd4278332c970ddc9122df0aadda5193c3a6edb87a"},
+ {file = "crc32c-2.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:c96e20f3ea5381126c0c66ad52857942dde9faa4ac4499086a55cc17d3d1bfdb"},
+ {file = "crc32c-2.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:53c4fd7e21a69df50611800f97db74f21aa2d4b122bc740a88385023c2e7bf0c"},
+ {file = "crc32c-2.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9dc5771634fafd1a548d7f84cfc0f1b8bb03f7fb4913614fb0f5998e2137af"},
+ {file = "crc32c-2.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b341f6f4a5222c171c04de2c2918212d8087c9c9955996c8af502169dd6ce376"},
+ {file = "crc32c-2.4-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3b192e3368faee25f63a544902f242a010974b12d3ee415871c3163b5d7547b"},
+ {file = "crc32c-2.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7f6775d6266f8f72734c2e8e2663262fef3a9ccc1953791099c21f19be58d5de"},
+ {file = "crc32c-2.4.tar.gz", hash = "sha256:d985c4d9b1a1fd16c593d83f8735a8e4e156790a95338a1e0b199aac51ca1e5e"},
+]
+
[[package]]
name = "cx-oracle"
version = "8.3.0"
@@ -767,4 +870,4 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[metadata]
lock-version = "2.0"
python-versions = "~3.10"
-content-hash = "364aa0f3743437380ffd40eede718b71cbd0999019803292845d157a6cc6cdab"
+content-hash = "51e1bd109c9b20d5e4a0044cbae4c75d9f24b91a52f527a5ab672ba3f0921c59"
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/fetcher_factory.py b/apps/cli/executables/pexable/productfetcher/productfetcher/fetcher_factory.py
index 9b3b15fdde14387d419dc3ab1ee3c47e90e30862..1423fb4f8085e3a746ce23a4f89c134ab043a46f 100644
--- a/apps/cli/executables/pexable/productfetcher/productfetcher/fetcher_factory.py
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/fetcher_factory.py
@@ -32,7 +32,6 @@ from .interfaces import FetcherFactory, FetchPlan, FileFetcher
from .locations import NgasFile, OracleXml
from .validators import Crc32Validator, SizeValidator, XmlWellFormedValidator
-
# pylint: disable=E0401, E0402, E1136, R0913
@@ -77,7 +76,7 @@ class ConfiguredFetcherFactory(FetcherFactory):
return (
RetryableFileFetcher(self.file_fetcher(file))
.and_validate_with(SizeValidator(file.size))
- .and_validate_with(Crc32Validator(file.checksum))
+ .and_validate_with(Crc32Validator(file.checksum, file.checksum_type))
)
def fetch_oracle_xml(self, file: OracleXml) -> FileFetcher:
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py b/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py
index 7ae7234788951843fe324c677bc3bc57b7ff667b..de1176f6ccddb343ece3c21532ebe27a414caf74 100644
--- a/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/locations.py
@@ -20,17 +20,18 @@ import http
import logging
# pylint: disable=E0239, E0401, E0402, E1136, R0201, R0902, R0903, R0913, W0613
-import requests
from enum import Enum
-from marshmallow import Schema, fields, post_load
-from marshmallow.decorators import validates_schema
from pathlib import Path
-from pycapo import CapoConfig
from typing import NamedTuple, Optional
+import requests
+from marshmallow import Schema, fields, post_load
+from marshmallow.decorators import validates_schema
from productfetcher.interfaces import FetcherFactory
+from pycapo import CapoConfig
+
from .exceptions import FetchError, LocatorServiceException
-from .interfaces import LocatedFile, LocationReport, Locator, FetchContext, FileFetcher
+from .interfaces import FetchContext, FileFetcher, LocatedFile, LocationReport, Locator
from .ngas import NgasConnection
logger = logging.getLogger("productfetcher.locations")
@@ -130,7 +131,7 @@ class NgasFile(LocatedFile):
checksum: int
checksum_type: str
version: int
- size: int
+ _size: int
server: NgasServer
_science_product_locator: str = None
diff --git a/apps/cli/executables/pexable/productfetcher/productfetcher/validators.py b/apps/cli/executables/pexable/productfetcher/productfetcher/validators.py
index 6adec3336ac51592e74dd98de97d27970cc88c49..dc56c31ed121bdc65bffcd549a3b7920d51533cc 100644
--- a/apps/cli/executables/pexable/productfetcher/productfetcher/validators.py
+++ b/apps/cli/executables/pexable/productfetcher/productfetcher/validators.py
@@ -34,10 +34,15 @@ We have three of these at the moment:
import binascii
import xml.sax
from pathlib import Path
+from typing import Callable
+
+import crc32c
from .exceptions import FileValidationFault
from .interfaces import FileValidator
+CRC32C_CHECKSUM_TYPE = "crc32c"
+
class SizeValidator(FileValidator):
"""
@@ -57,24 +62,31 @@ class SizeValidator(FileValidator):
class Crc32Validator(FileValidator):
"""
- Validates the cksum of a file. This is an algorithm used by NGAS.
- If the content of the file generates the same cksum we expect,
- the file passes validation.
+ Validates the checksum of a file using an algorithm specified by NGAS.
+ Supports both CRC32C, in use at NAASC, and CRC32, used by SSA.
+ CRC32C is used when `checksum_type == CRC32C_CHECKSUM_TYPE`, CRC32 by default.
+ If the content of the file generates the same cksum we expect, the file passes validation.
"""
- def __init__(self, checksum: int):
+ checksum_algorithm: Callable[[bytes, int], int]
+
+ def __init__(self, checksum: int, checksum_type: str):
self.checksum = checksum
+ if checksum_type == CRC32C_CHECKSUM_TYPE:
+ self.checksum_algorithm = crc32c.crc32c
+ else:
+ self.checksum_algorithm = binascii.crc32
def validate(self, path: Path):
# there are a couple ways to do this, another is with the zlib module
# the important thing is reading this a chunk at a time
- checksum = binascii.crc32(b"")
+ checksum = self.checksum_algorithm(b"", 0)
with path.open("rb") as content:
# read1() guarantees that only a single system read() call is made
# this should be more efficient than reading the entire thing into memory at once
# 0 is returned when we run out of stuff to read
while len(chunk := content.read1()) != 0:
- checksum = binascii.crc32(chunk, checksum)
+ checksum = self.checksum_algorithm(chunk, checksum)
# the magic bitmath here is courtesy of the Python documentation:
# Python 2 generated signed numbers here, but Python 3 generates unsigned numbers instead
diff --git a/apps/cli/executables/pexable/productfetcher/pyproject.toml b/apps/cli/executables/pexable/productfetcher/pyproject.toml
index 37a1f4015e82f0cb4d1b7cf8d15b114135464384..c239f56b04479963033f2d1abe75450a1a692117 100644
--- a/apps/cli/executables/pexable/productfetcher/pyproject.toml
+++ b/apps/cli/executables/pexable/productfetcher/pyproject.toml
@@ -20,6 +20,7 @@ psycopg2-binary = "2.9.9"
requests = "2.31.0"
tqdm = "4.66.0"
urllib3 = "<2.0.0"
+crc32c = "2.4"
[tool.poetry.group.test.dependencies]
requests-mock = "1.11.0"
diff --git a/apps/cli/executables/pexable/productfetcher/tests/test_fetcher_factory.py b/apps/cli/executables/pexable/productfetcher/tests/test_fetcher_factory.py
index 365f7dc692816439c107fe6418a86e665a1640af..138d6aa209e65dc09fa9730011e4f286d06a33a3 100644
--- a/apps/cli/executables/pexable/productfetcher/tests/test_fetcher_factory.py
+++ b/apps/cli/executables/pexable/productfetcher/tests/test_fetcher_factory.py
@@ -28,7 +28,6 @@ from productfetcher.fetchers import (
)
from test_product_fetcher import FakeFetchContext
-
# pylint: disable=E0239, E0401, E0402, R0201, R0903
@@ -37,6 +36,7 @@ class FakeFile(NamedTuple):
the_size: int
checksum: int
+ checksum_type: str
def size(self) -> int:
return self.the_size
@@ -52,7 +52,7 @@ def test_configured_fetcher_factory():
:return:
"""
- fake_file = FakeFile(239823983, -329831113)
+ fake_file = FakeFile(239823983, -329831113, "ngamsGenCrc32")
fake_context = FakeFetchContext()
# let's try a few combinations and make sure we find reasonable things
diff --git a/apps/cli/executables/pexable/productfetcher/tests/test_validators.py b/apps/cli/executables/pexable/productfetcher/tests/test_validators.py
index 12232477ed8be5dbcee0d85a2f7770698a4e6371..50e46ae6dea1eaa5954538027683a91dabdb0dfa 100644
--- a/apps/cli/executables/pexable/productfetcher/tests/test_validators.py
+++ b/apps/cli/executables/pexable/productfetcher/tests/test_validators.py
@@ -21,6 +21,7 @@ import pathlib
import pytest
from productfetcher.exceptions import FileValidationFault
from productfetcher.validators import (
+ CRC32C_CHECKSUM_TYPE,
Crc32Validator,
SizeValidator,
XmlWellFormedValidator,
@@ -57,16 +58,21 @@ def test_checksum_validator(resource_path_root: pathlib.Path, capsys):
:param capsys:
:return:
"""
+ random_junk_path = resource_path_root / "validators" / "random-junk"
# this should work just fine
- Crc32Validator(457325410).validate(resource_path_root / "validators" / "random-junk")
+ Crc32Validator(457325410, "ngamsGenCrc32").validate(random_junk_path)
+ Crc32Validator(457325410, "other-cksum-type").validate(random_junk_path)
+ Crc32Validator(1374108510, CRC32C_CHECKSUM_TYPE).validate(random_junk_path)
# this should raise an error
with pytest.raises(FileValidationFault):
- Crc32Validator(457325411).validate(resource_path_root / "validators" / "random-junk")
+ Crc32Validator(457325411, "ngamsGenCrc32").validate(random_junk_path)
+ with pytest.raises(FileValidationFault):
+ Crc32Validator(1374108511, CRC32C_CHECKSUM_TYPE).validate(random_junk_path)
# so should this
with pytest.raises(FileValidationFault):
- Crc32Validator(-1).validate(resource_path_root / "validators" / "random-junk")
+ Crc32Validator(-1, "ngamsGenCrc32").validate(random_junk_path)
capsys.readouterr()