From 71fc558fc2985d33842a1c6d5f6b436b79c345e8 Mon Sep 17 00:00:00 2001 From: Triyan Mukherjee Date: Fri, 21 Aug 2026 03:37:50 +0530 Subject: [PATCH 1/7] feat: Optimize the clone process to use threadpool --- .gitignore | 1 + poetry.lock | 118 +++++++++++++++++++++++++--- poetry.toml | 2 + pokeapi_ditto/commands/analyze.py | 37 ++++----- pokeapi_ditto/commands/clone.py | 110 ++++++++++++++++++++------ pokeapi_ditto/commands/models.py | 65 ++++++++------- pokeapi_ditto/commands/transform.py | 14 ++-- pokeapi_ditto/common.py | 18 ----- pokeapi_ditto/main.py | 11 +-- pyproject.toml | 6 +- 10 files changed, 265 insertions(+), 117 deletions(-) create mode 100644 poetry.toml diff --git a/.gitignore b/.gitignore index f28927f3e..6c5907d3d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .idea/ +.vscode/ *.py[cod] diff --git a/poetry.lock b/poetry.lock index 0359892d0..99469ad88 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "black" @@ -6,6 +6,7 @@ version = "24.10.0" description = "The uncompromising code formatter." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "black-24.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6668650ea4b685440857138e5fe40cde4d652633b1bdffc62933d0db4ed9812"}, {file = "black-24.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1c536fcf674217e87b8cc3657b81809d3c085d7bf3ef262ead700da345bfa6ea"}, @@ -52,6 +53,7 @@ version = "2026.2.25" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "certifi-2026.2.25-py3-none-any.whl", hash = "sha256:027692e4402ad994f1c42e52a4997a9763c646b73e4096e4d5d6db8af1d6f0fa"}, {file = "certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7"}, @@ -63,6 +65,7 @@ version = "3.4.5" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "charset_normalizer-3.4.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4167a621a9a1a986c73777dbc15d4b5eac8ac5c10393374109a343d4013ec765"}, {file = "charset_normalizer-3.4.5-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3f64c6bf8f32f9133b668c7f7a7cbdbc453412bc95ecdbd157f3b1e377a92990"}, @@ -185,6 +188,7 @@ version = "8.3.1" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -199,10 +203,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "platform_system == \"Windows\"", dev = "sys_platform == \"win32\" or platform_system == \"Windows\""} [[package]] name = "exceptiongroup" @@ -210,6 +216,8 @@ version = "1.3.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, @@ -227,6 +235,7 @@ version = "7.3.0" description = "the modular source code checker: pep8 pyflakes and co" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "flake8-7.3.0-py2.py3-none-any.whl", hash = "sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e"}, {file = "flake8-7.3.0.tar.gz", hash = "sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872"}, @@ -243,6 +252,7 @@ version = "1.3.0" description = "GenSON is a powerful, user-friendly JSON Schema generator." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "genson-1.3.0-py3-none-any.whl", hash = "sha256:468feccd00274cc7e4c09e84b08704270ba8d95232aa280f65b986139cec67f7"}, {file = "genson-1.3.0.tar.gz", hash = "sha256:e02db9ac2e3fd29e65b5286f7135762e2cd8a986537c075b06fc5f1517308e37"}, @@ -254,6 +264,7 @@ version = "3.11" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea"}, {file = "idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902"}, @@ -268,6 +279,7 @@ version = "2.3.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, @@ -279,6 +291,7 @@ version = "5.13.2" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" +groups = ["dev"] files = [ {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, @@ -293,6 +306,7 @@ version = "0.7.0" description = "McCabe checker, plugin for flake8" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, @@ -304,6 +318,7 @@ version = "6.7.1" description = "multidict implementation" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c93c3db7ea657dd4637d57e74ab73de31bccefe144d3d4ce370052035bc85fb5"}, {file = "multidict-6.7.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:974e72a2474600827abaeda71af0c53d9ebbc3c2eb7da37b37d7829ae31232d8"}, @@ -462,19 +477,85 @@ version = "1.1.0" description = "Type system extensions for programs checked with the mypy type checker." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, ] [[package]] -name = "odictliteral" -version = "1.0.0" -description = "A tidier way of coding literal OrderedDicts" +name = "orjson" +version = "3.12.0" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" optional = false -python-versions = "*" +python-versions = ">=3.10" +groups = ["main"] files = [ - {file = "odictliteral-1.0.0.tar.gz", hash = "sha256:88405c7fab7ff7a54c7b9fac9fd69264e526b0024b8265bc042ba3a797f0c161"}, + {file = "orjson-3.12.0-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:747843254519dd43b93eee3153a19e5a509334320c4d2f823ec879232db5c796"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7c2ad193c8004254f34b499f3bd2c80f043d10754aff2b38f93da574f4883f98"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:bc7a872f03522d90e0429e6c0c5cd23084f767bedcb4c58048eec19294613344"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18a87929f31d94a77f7dc93cf527e91f39ce7fe7813d588a4de2507efd32a387"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9683ee9ea0659da64f36574ef675b8a86330c34c19ea75db1fb93c3ff99e0ef"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:103b5db66aa53c1f9e88c2524be4f383e831ba7dfd5f9f5af6336a177c622f11"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd57d79aefa3f84eec851d6de7a366795b9345cfaf17f82b4820430a7a5fa241"}, + {file = "orjson-3.12.0-cp310-cp310-win32.whl", hash = "sha256:3dbce9b6b3074b31a5d5dd322a9c4e5b16f206091ece4194c2e36952847a105e"}, + {file = "orjson-3.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bb17a06f9bd15237b3216c044209fe92597379124018cfc196fbb846cde64df"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a94f0f0c6fcbb2b5bd9734c57a489c7584a732bbdf04a39e8c83b861e9d03e92"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a696529ec96a90d9a5f9570207efe403c8b08f8e4aa2783ee3403511e2fdfa10"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:e4ac5059baab4b3acbd99485de019ff8cda0fdf34b61fa74f7197a53db78bfe8"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e29957429c35bbb5a185a119c523aa2428b7bbf1a293724c7b9375ed8f892a3"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce0166feb0a737ab84f598c9a338cbc0b764a036617aa686194f53c7eba0c3e"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9caf3d09f47c3c70c4451ada20ef9bc4a4cdffa26f49862cf0a253b329aae2d5"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b9dca132b1fda5565088e65a6b6e742285e0aeceb6fae549fa8863e16c7d3998"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a791f793b287bbc135b8e87c34e35c8bfc693e2a8a620fab1ae682b925f9a32e"}, + {file = "orjson-3.12.0-cp311-cp311-win32.whl", hash = "sha256:31ed278a36304390adc3eec5d7f6fd593a7c3e99e5a06cd07866396c4b1b4710"}, + {file = "orjson-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb2539159dfe8d371914f354360fa50e4a577cc89222a3828b9650a5e5040252"}, + {file = "orjson-3.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:61318b6de893c7a9d9f3e5ecbadccbfc26a7eb417ccc7bbf0771de3b4d72f868"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873"}, + {file = "orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5"}, + {file = "orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a"}, + {file = "orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222"}, + {file = "orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1"}, + {file = "orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2"}, + {file = "orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f"}, + {file = "orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92"}, + {file = "orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed"}, + {file = "orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a"}, + {file = "orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55"}, + {file = "orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578"}, + {file = "orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc"}, + {file = "orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5"}, ] [[package]] @@ -483,6 +564,7 @@ version = "26.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529"}, {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, @@ -494,6 +576,7 @@ version = "1.0.4" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723"}, {file = "pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645"}, @@ -511,6 +594,7 @@ version = "4.9.4" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.10" +groups = ["dev"] files = [ {file = "platformdirs-4.9.4-py3-none-any.whl", hash = "sha256:68a9a4619a666ea6439f2ff250c12a853cd1cbd5158d258bd824a7df6be2f868"}, {file = "platformdirs-4.9.4.tar.gz", hash = "sha256:1ec356301b7dc906d83f371c8f487070e99d3ccf9e501686456394622a01a934"}, @@ -522,6 +606,7 @@ version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -537,6 +622,7 @@ version = "0.4.1" description = "Accelerated property cache" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c2d1fa3201efaf55d730400d945b5b3ab6e672e100ba0f9a409d950ab25d7db"}, {file = "propcache-0.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1eb2994229cc8ce7fe9b3db88f5465f5fd8651672840b2e426b88cdb1a30aac8"}, @@ -668,6 +754,7 @@ version = "2.14.0" description = "Python style guide checker" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pycodestyle-2.14.0-py2.py3-none-any.whl", hash = "sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d"}, {file = "pycodestyle-2.14.0.tar.gz", hash = "sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783"}, @@ -679,6 +766,7 @@ version = "3.4.0" description = "passive checker of Python programs" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pyflakes-3.4.0-py2.py3-none-any.whl", hash = "sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f"}, {file = "pyflakes-3.4.0.tar.gz", hash = "sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58"}, @@ -690,6 +778,7 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -704,6 +793,7 @@ version = "8.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79"}, {file = "pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01"}, @@ -727,6 +817,7 @@ version = "2.32.5" description = "Python HTTP for Humans." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6"}, {file = "requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf"}, @@ -748,6 +839,8 @@ version = "2.4.0" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b5ef256a3fd497d4973c11bf142e9ed78b150d36f5773f1ca6088c230ffc5867"}, {file = "tomli-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5572e41282d5268eb09a697c89a7bee84fae66511f87533a6f88bd2f7b652da9"}, @@ -804,6 +897,7 @@ version = "4.67.3" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf"}, {file = "tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb"}, @@ -825,6 +919,8 @@ version = "4.15.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] +markers = "python_version == \"3.10\"" files = [ {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, @@ -836,16 +932,17 @@ version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] [package.extras] -brotli = ["brotli (>=1.2.0)", "brotlicffi (>=1.2.0.0)"] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] -zstd = ["backports-zstd (>=1.0.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [[package]] name = "yarl" @@ -853,6 +950,7 @@ version = "1.23.0" description = "Yet another URL library" optional = false python-versions = ">=3.10" +groups = ["main"] files = [ {file = "yarl-1.23.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cff6d44cb13d39db2663a22b22305d10855efa0fa8015ddeacc40bc59b9d8107"}, {file = "yarl-1.23.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c53f8347cd4200f0d70a48ad059cabaf24f5adc6ba08622a23423bc7efa10d"}, @@ -990,6 +1088,6 @@ multidict = ">=4.0" propcache = ">=0.2.1" [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.10" -content-hash = "5e42cd7b9f10946086ca1d4d9164f84d6e551073730b83ba0422bd0d83958e92" +content-hash = "59129be34aa1b214fd23336cb49e96c7a1d650668e3fe78f4e9b415ff8b34a0f" diff --git a/poetry.toml b/poetry.toml new file mode 100644 index 000000000..ab1033bd3 --- /dev/null +++ b/poetry.toml @@ -0,0 +1,2 @@ +[virtualenvs] +in-project = true diff --git a/pokeapi_ditto/commands/analyze.py b/pokeapi_ditto/commands/analyze.py index c6d08479d..f354c7489 100644 --- a/pokeapi_ditto/commands/analyze.py +++ b/pokeapi_ditto/commands/analyze.py @@ -1,15 +1,14 @@ import glob -import json import os import re from pathlib import Path from typing import Dict, List, TypeVar +import orjson from genson import SchemaBuilder from tqdm import tqdm from pokeapi_ditto.commands.models import COMMON_MODELS -from pokeapi_ditto.common import from_path T = TypeVar("T") @@ -43,39 +42,40 @@ def do_analyze(data_dir: str): if not schema_path.exists(): schema_path.mkdir(parents=True) - @from_path(api_path) def get_schema_paths() -> List[Path]: return sorted( { - Path(*[re.sub("^[0-9]+$", "$id", part) for part in path.parts]) - for path in Path(".").glob("**/*.json") + Path( + *[ + re.sub("^[0-9]+$", "$id", part) + for part in path.relative_to(api_path).parts + ] + ) + for path in api_path.glob("**/*.json") } ) - @from_path(api_path) def gen_single_schema(path: Path) -> SchemaBuilder: - glob_exp = os.path.join( - *["*" if part == "$id" else part for part in path.parts] + glob_exp = str( + api_path + / os.path.join(*["*" if part == "$id" else part for part in path.parts]) ) file_names = list(glob.iglob(glob_exp, recursive=True)) schema = SchemaBuilder() for file_name in tqdm(file_names, desc=str(path.parent)): - with open(file_name) as f: - schema.add_object(json.load(f)) + with open(file_name, "rb") as f: + schema.add_object(orjson.loads(f.read())) return schema - @from_path(schema_path) def gen_schemas(paths: List[Path]): for path in tqdm(paths): - if not path.parent.exists(): - os.makedirs(path.parent) + out_path = schema_path / path + out_path.parent.mkdir(parents=True, exist_ok=True) schema = gen_single_schema(path).to_schema() for name, model in COMMON_MODELS.items(): schema = _replace_common_model(schema, name, model) - with path.open("w") as f: - f.write(json.dumps(schema, indent=4, sort_keys=True)) + out_path.write_bytes(orjson.dumps(schema, option=orjson.OPT_INDENT_2)) - @from_path(data_path) def save_common_schemas(): for name, model in COMMON_MODELS.items(): schema_builder = SchemaBuilder() @@ -84,8 +84,9 @@ def save_common_schemas(): if name.endswith("resource_list.json"): schema["properties"]["next"]["type"] = ["null", "string"] schema["properties"]["previous"]["type"] = ["null", "string"] - with Path(name).relative_to(Path(name).root).open("w") as f: - f.write(json.dumps(schema, indent=4, sort_keys=True)) + out_file = data_path / Path(name).relative_to(Path(name).root) + out_file.parent.mkdir(parents=True, exist_ok=True) + out_file.write_bytes(orjson.dumps(schema, option=orjson.OPT_INDENT_2)) gen_schemas(get_schema_paths()) save_common_schemas() diff --git a/pokeapi_ditto/commands/clone.py b/pokeapi_ditto/commands/clone.py index 6ac83f7a6..a4e073bc5 100644 --- a/pokeapi_ditto/commands/clone.py +++ b/pokeapi_ditto/commands/clone.py @@ -1,33 +1,68 @@ -import json import os -from multiprocessing import Pool +from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from signal import SIG_IGN, SIGINT, signal -from typing import Any, Callable, List, Tuple +from typing import Any, Callable, List, NamedTuple, Tuple +import orjson import requests +from requests.adapters import HTTPAdapter from tqdm import tqdm +from urllib3.util.retry import Retry from yarl import URL -def _do_in_parallel(worker: Callable, data: List, desc: str) -> None: - cpus = os.cpu_count() - 1 - pool = Pool(cpus, initializer=lambda: signal(SIGINT, SIG_IGN)) - try: - for _ in tqdm( - pool.imap_unordered(worker, data), total=len(data), desc=f"{desc} ({cpus}x)", mininterval=1 - ): - pass - except KeyboardInterrupt as interrupt: - pool.terminate() - pool.join() - raise interrupt +class RequestTimeout(NamedTuple): + connect: int + read: int + + +_REQUEST_TIMEOUT = RequestTimeout(connect=5, read=30) + + +def _calculate_max_workers() -> int: + """ + Derive client thread count from co-located server capacity. + + https://github.com/PokeAPI/pokeapi/blob/master/gunicorn.conf.py + + Assumes both this client and the target server run on the same machine, + and the server uses gunicorn's default worker formula: 2 * cpu_count. + We target 1.5x the server's worker count to keep the request pipeline + saturated (accounting for network/IO round-trip slack) without starving + the server of CPU time. Capped at 24 to bound memory and fd usage. + """ + cpu = os.cpu_count() or 4 + server_workers = 2 * cpu # gunicorn default: 2 * CPU count + client_threads = int(server_workers * 1.5) # 1.5x to fill the pipeline + return min(max(4, client_threads), 24) + + +_MAX_WORKERS = _calculate_max_workers() + + +def _do_in_parallel(worker: Callable[[Tuple[str, str]], None], data: List[Tuple[str, str]], desc: str) -> None: + with ThreadPoolExecutor(max_workers=_MAX_WORKERS) as executor: + futures = [executor.submit(worker, item) for item in data] + try: + for future in tqdm( + as_completed(futures), + total=len(futures), + desc=f"{desc} ({_MAX_WORKERS}T)", + mininterval=1, + position=1, + leave=False, + ): + future.result() + except KeyboardInterrupt: + executor.shutdown(wait=False, cancel_futures=True) + raise class Cloner: _src_url: URL _dest_dir: Path + _session: requests.Session def __init__(self, src_url: str, dest_dir: str): if src_url.endswith("/"): @@ -37,20 +72,45 @@ def __init__(self, src_url: str, dest_dir: str): self._src_url = URL(src_url) self._dest_dir = Path(dest_dir) + self._session = self._build_session() + + @staticmethod + def _build_session() -> requests.Session: + session = requests.Session() + retry = Retry( + total=3, + backoff_factor=0.5, + status_forcelist=[500, 502, 503, 504], + allowed_methods=["GET"], + ) + adapter = HTTPAdapter( + pool_connections=_MAX_WORKERS, + pool_maxsize=_MAX_WORKERS, + max_retries=retry, + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session def _crawl(self, url: URL, save: bool = True) -> Any: try: - data = requests.get(url).json() - except json.JSONDecodeError as err: + response = self._session.get(str(url), timeout=_REQUEST_TIMEOUT) + response.raise_for_status() + data = orjson.loads(response.content) + except requests.RequestException as e: + tqdm.write(f"Request failure: {url} ({e})") + return None + except orjson.JSONDecodeError: tqdm.write(f"JSON decode failure: {url}") return None if save: - out_data = json.dumps(data, indent=4, sort_keys=True) - out_data = out_data.replace(str(self._src_url), "") + out_data = orjson.dumps(data, option=orjson.OPT_INDENT_2) + src_url_bytes = str(self._src_url).encode("utf-8") + out_data = out_data.replace(src_url_bytes, b"") file = self._dest_dir.joinpath((url / "index.json").path[1:]) file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(out_data) + file.write_bytes(out_data) return data @@ -65,7 +125,9 @@ def _crawl_resource_list(self, url: URL) -> List[URL]: count = payload["count"] full_url = url.with_query({"limit": count, "offset": 0}) resource_list = self._crawl(full_url) - return [URL(resource_ref["url"]) for resource_ref in resource_list["results"]] + return [ + URL(resource_ref["url"]) for resource_ref in resource_list["results"] + ] else: self._crawl(url) return [] @@ -85,7 +147,7 @@ def clone_endpoint(self, endpoint: str): def clone_all(self) -> None: resource_lists = self._crawl_index() - for res_list_url in tqdm(resource_lists, desc="clone"): + for res_list_url in tqdm(resource_lists, desc="clone", position=0): endpoint = res_list_url.parent.name self.clone_endpoint(endpoint) @@ -98,6 +160,6 @@ def do_clone(src_url: str, dest_dir: str, select: List[str]) -> None: for sel in select: if "/" in sel: - cloner.clone_single(tuple(filter(None, sel.split("/")))[0:2]) + cloner.clone_single(tuple(filter(None, sel.split("/")))[0:2]) # pyright: ignore[reportArgumentType] else: cloner.clone_endpoint(sel) diff --git a/pokeapi_ditto/commands/models.py b/pokeapi_ditto/commands/models.py index a7c533d02..90472ad41 100644 --- a/pokeapi_ditto/commands/models.py +++ b/pokeapi_ditto/commands/models.py @@ -1,117 +1,116 @@ -from collections import OrderedDict +# noinspection DuplicatedCode +from typing import Any -from odictliteral import odict -# noinspection DuplicatedCode -COMMON_MODELS: OrderedDict = odict[ - "/schema/v2/api_resource.json":{ +COMMON_MODELS: dict[str, Any] = { + "/schema/v2/api_resource.json": { "properties": {"url": {"type": "string"}}, "required": ["url"], "type": "object", }, - "/schema/v2/named_api_resource.json":{ + "/schema/v2/named_api_resource.json": { "properties": {"name": {"type": "string"}, "url": {"type": "string"}}, "required": ["name", "url"], "type": "object", }, - "/schema/v2/api_resource_list.json":{ + "/schema/v2/api_resource_list.json": { "properties": { "count": {"type": "integer"}, "next": {"type": "null"}, "previous": {"type": "null"}, "results": { - "items": {"$ref": f"/schema/v2/api_resource.json"}, + "items": {"$ref": "/schema/v2/api_resource.json"}, "type": "array", }, }, "required": ["count", "next", "previous", "results"], "type": "object", }, - "/schema/v2/named_api_resource_list.json":{ + "/schema/v2/named_api_resource_list.json": { "properties": { "count": {"type": "integer"}, "next": {"type": "null"}, "previous": {"type": "null"}, "results": { - "items": {"$ref": f"/schema/v2/named_api_resource.json"}, + "items": {"$ref": "/schema/v2/named_api_resource.json"}, "type": "array", }, }, "required": ["count", "next", "previous", "results"], "type": "object", }, - "/schema/v2/description.json":{ + "/schema/v2/description.json": { "properties": { "description": {"type": "string"}, - "language": {"$ref": f"/schema/v2/named_api_resource.json"}, + "language": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["description", "language"], "type": "object", }, - "/schema/v2/effect.json":{ + "/schema/v2/effect.json": { "properties": { "effect": {"type": "string"}, - "language": {"$ref": f"/schema/v2/named_api_resource.json"}, + "language": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["effect", "language"], "type": "object", }, - "/schema/v2/encounter.json":{ + "/schema/v2/encounter.json": { "properties": { "chance": {"type": "integer"}, "condition_values": { - "items": {"$ref": f"/schema/v2/named_api_resource.json"}, + "items": {"$ref": "/schema/v2/named_api_resource.json"}, "type": "array", }, "max_level": {"type": "integer"}, - "method": {"$ref": f"/schema/v2/named_api_resource.json"}, + "method": {"$ref": "/schema/v2/named_api_resource.json"}, "min_level": {"type": "integer"}, }, "required": ["chance", "condition_values", "max_level", "method", "min_level"], "type": "object", }, - "/schema/v2/flavor_text.json":{ + "/schema/v2/flavor_text.json": { "properties": { "flavor_text": {"type": "string"}, - "language": {"$ref": f"/schema/v2/named_api_resource.json"}, + "language": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["flavor_text", "language"], "type": "object", }, - "/schema/v2/generation_game_index.json":{ + "/schema/v2/generation_game_index.json": { "properties": { "game_index": {"type": "integer"}, - "generation": {"$ref": f"/schema/v2/named_api_resource.json"}, + "generation": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["game_index", "generation"], "type": "object", }, - "/schema/v2/machine_version_detail.json":{ + "/schema/v2/machine_version_detail.json": { "properties": { - "machine": {"$ref": f"/schema/v2/api_resource.json"}, - "version_group": {"$ref": f"/schema/v2/named_api_resource.json"}, + "machine": {"$ref": "/schema/v2/api_resource.json"}, + "version_group": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["machine", "version_group"], "type": "object", }, - "/schema/v2/name.json":{ + "/schema/v2/name.json": { "properties": { - "language": {"$ref": f"/schema/v2/named_api_resource.json"}, + "language": {"$ref": "/schema/v2/named_api_resource.json"}, "name": {"type": "string"}, }, "required": ["language", "name"], "type": "object", }, - "/schema/v2/verbose_effect.json":{ + "/schema/v2/verbose_effect.json": { "properties": { "effect": {"type": "string"}, - "language": {"$ref": f"/schema/v2/named_api_resource.json"}, + "language": {"$ref": "/schema/v2/named_api_resource.json"}, "short_effect": {"type": "string"}, }, "required": ["effect", "language", "short_effect"], "type": "object", }, - "/schema/v2/version_encounter_detail.json":{ + "/schema/v2/version_encounter_detail.json": { "properties": { "encounter_details": { "items": {"$ref": "/schema/v2/encounter.json"}, @@ -123,15 +122,15 @@ "required": ["encounter_details", "max_chance", "version"], "type": "object", }, - "/schema/v2/version_game_index.json":{ + "/schema/v2/version_game_index.json": { "properties": { "game_index": {"type": "integer"}, - "version": {"$ref": f"/schema/v2/named_api_resource.json"}, + "version": {"$ref": "/schema/v2/named_api_resource.json"}, }, "required": ["game_index", "version"], "type": "object", }, - "/schema/v2/version_group_flavor_text.json":{ + "/schema/v2/version_group_flavor_text.json": { "properties": { "language": {"$ref": "/schema/v2/named_api_resource.json"}, "text": {"type": "string"}, @@ -140,4 +139,4 @@ "required": ["language", "text", "version_group"], "type": "object", }, -] +} diff --git a/pokeapi_ditto/commands/transform.py b/pokeapi_ditto/commands/transform.py index f3f0556d7..cfea35add 100644 --- a/pokeapi_ditto/commands/transform.py +++ b/pokeapi_ditto/commands/transform.py @@ -1,7 +1,7 @@ -import json from pathlib import Path from typing import Any, Dict +import orjson from tqdm import tqdm from pokeapi_ditto.common import apply_base_url @@ -18,7 +18,7 @@ def _is_id(s: str): def _dump(path: Path, content: Any): if not path.parent.exists(): path.parent.mkdir(parents=True) - path.write_text(json.dumps(content, sort_keys=True, indent=4)) + path.write_bytes(orjson.dumps(content, option=orjson.OPT_INDENT_2)) # TODO: blow all this up and make it good @@ -27,9 +27,9 @@ def _dump(path: Path, content: Any): # clone.py is a cleaner model to follow -def do_transform(src_dir: str, dest_dir: str, base_url: str): - src_dir: Path = Path(src_dir) - dest_dir: Path = Path(dest_dir) +def do_transform(source_directory: str, destination_directory: str, base_url: str): + src_dir: Path = Path(source_directory) + dest_dir: Path = Path(destination_directory) if base_url.endswith("/"): base_url = base_url[:-1] @@ -40,7 +40,7 @@ def do_transform(src_dir: str, dest_dir: str, base_url: str): src_paths = src_dir.glob("**/*.json") for src_path in tqdm(list(src_paths)): - content: Dict = json.loads(apply_base_url(src_path.read_text(), base_url)) + content: Dict[str, Any] = orjson.loads(apply_base_url(src_path.read_text(), base_url)) # all files dest_path = dest_dir.joinpath(src_path.relative_to(src_dir)) @@ -58,7 +58,7 @@ def do_transform(src_dir: str, dest_dir: str, base_url: str): and dest_path.parent.name == "encounters" ): pokemon_path = src_path.parent.parent.joinpath("index.json") - name = json.loads(pokemon_path.read_text())["name"] + name = orjson.loads(pokemon_path.read_bytes())["name"] dest_path = dest_path.parent.parent.parent.joinpath( name, "encounters", "index.json" ) diff --git a/pokeapi_ditto/common.py b/pokeapi_ditto/common.py index d424f30ec..99e8cb889 100644 --- a/pokeapi_ditto/common.py +++ b/pokeapi_ditto/common.py @@ -1,22 +1,4 @@ -import os import re -from pathlib import Path - - -def from_path(target_path: Path) -> callable: - target_path = target_path.absolute() - - def func_decorator(func: callable) -> callable: - def func_wrapper(*args, **kwargs): - cwd = os.getcwd() - os.chdir(str(target_path)) - result = func(*args, **kwargs) - os.chdir(cwd) - return result - - return func_wrapper - - return func_decorator def apply_base_url(data: str, base_url: str) -> str: diff --git a/pokeapi_ditto/main.py b/pokeapi_ditto/main.py index b58315c4b..7669832c3 100644 --- a/pokeapi_ditto/main.py +++ b/pokeapi_ditto/main.py @@ -1,6 +1,7 @@ import argparse -import sys import importlib.metadata +import sys +from typing import Any from pokeapi_ditto.commands import analyze, clone, transform @@ -11,7 +12,7 @@ def __init__(self): parser.add_argument( "--version", action="version", - version=importlib.metadata.version('pokeapi-ditto'), + version=importlib.metadata.version("pokeapi-ditto"), ) subparsers = parser.add_subparsers(dest="command") @@ -40,15 +41,15 @@ def __init__(self): getattr(self, command)(args) @staticmethod - def clone(args): + def clone(args: Any): clone.do_clone(**args) @staticmethod - def transform(args): + def transform(args: Any): transform.do_transform(**args) @staticmethod - def analyze(args): + def analyze(args: Any): analyze.do_analyze(**args) diff --git a/pyproject.toml b/pyproject.toml index 095454522..643f54c69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,8 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] [tool.poetry.scripts] @@ -28,10 +30,10 @@ python = "^3.10" requests = "^2.32" genson = "^1.3" tqdm = "^4.67" -odictliteral = "^1.0" +orjson = "^3.10" yarl = "^1.23" -[tool.poetry.dev-dependencies] +[tool.poetry.group.dev.dependencies] pytest = "^8.4" flake8 = "^7.3" black = "^24.10" From f4831c9d6e3b9f0566461da999520221d2cb2886 Mon Sep 17 00:00:00 2001 From: Triyan Mukherjee Date: Fri, 21 Aug 2026 03:43:56 +0530 Subject: [PATCH 2/7] lint: run black, isort and flake8 --- .flake8 | 3 ++- poetry.toml | 2 -- pokeapi_ditto/commands/clone.py | 10 +++++++--- pokeapi_ditto/commands/models.py | 1 - pokeapi_ditto/commands/transform.py | 4 +++- 5 files changed, 12 insertions(+), 8 deletions(-) delete mode 100644 poetry.toml diff --git a/.flake8 b/.flake8 index 04c854c33..17535f68c 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,6 @@ [flake8] # copy changes to .stickler.yml max-complexity = 15 -ignore = W,E # use black for formatting +# use black for formatting +ignore = W,E exclude = .venv, venv, .git, __pycache__, dist, data diff --git a/poetry.toml b/poetry.toml deleted file mode 100644 index ab1033bd3..000000000 --- a/poetry.toml +++ /dev/null @@ -1,2 +0,0 @@ -[virtualenvs] -in-project = true diff --git a/pokeapi_ditto/commands/clone.py b/pokeapi_ditto/commands/clone.py index a4e073bc5..a4bf22462 100644 --- a/pokeapi_ditto/commands/clone.py +++ b/pokeapi_ditto/commands/clone.py @@ -32,7 +32,7 @@ def _calculate_max_workers() -> int: the server of CPU time. Capped at 24 to bound memory and fd usage. """ cpu = os.cpu_count() or 4 - server_workers = 2 * cpu # gunicorn default: 2 * CPU count + server_workers = 2 * cpu # gunicorn default: 2 * CPU count client_threads = int(server_workers * 1.5) # 1.5x to fill the pipeline return min(max(4, client_threads), 24) @@ -40,7 +40,9 @@ def _calculate_max_workers() -> int: _MAX_WORKERS = _calculate_max_workers() -def _do_in_parallel(worker: Callable[[Tuple[str, str]], None], data: List[Tuple[str, str]], desc: str) -> None: +def _do_in_parallel( + worker: Callable[[Tuple[str, str]], None], data: List[Tuple[str, str]], desc: str +) -> None: with ThreadPoolExecutor(max_workers=_MAX_WORKERS) as executor: futures = [executor.submit(worker, item) for item in data] try: @@ -160,6 +162,8 @@ def do_clone(src_url: str, dest_dir: str, select: List[str]) -> None: for sel in select: if "/" in sel: - cloner.clone_single(tuple(filter(None, sel.split("/")))[0:2]) # pyright: ignore[reportArgumentType] + cloner.clone_single( + tuple(filter(None, sel.split("/")))[0:2] # pyright: ignore[reportArgumentType] + ) else: cloner.clone_endpoint(sel) diff --git a/pokeapi_ditto/commands/models.py b/pokeapi_ditto/commands/models.py index 90472ad41..0804eff5e 100644 --- a/pokeapi_ditto/commands/models.py +++ b/pokeapi_ditto/commands/models.py @@ -1,7 +1,6 @@ # noinspection DuplicatedCode from typing import Any - COMMON_MODELS: dict[str, Any] = { "/schema/v2/api_resource.json": { "properties": {"url": {"type": "string"}}, diff --git a/pokeapi_ditto/commands/transform.py b/pokeapi_ditto/commands/transform.py index cfea35add..45b53ca88 100644 --- a/pokeapi_ditto/commands/transform.py +++ b/pokeapi_ditto/commands/transform.py @@ -40,7 +40,9 @@ def do_transform(source_directory: str, destination_directory: str, base_url: st src_paths = src_dir.glob("**/*.json") for src_path in tqdm(list(src_paths)): - content: Dict[str, Any] = orjson.loads(apply_base_url(src_path.read_text(), base_url)) + content: Dict[str, Any] = orjson.loads( + apply_base_url(src_path.read_text(), base_url) + ) # all files dest_path = dest_dir.joinpath(src_path.relative_to(src_dir)) From d117bb9c67d57cac581e8392116ac02b7be7b636 Mon Sep 17 00:00:00 2001 From: Triyan Mukherjee Date: Fri, 21 Aug 2026 03:53:44 +0530 Subject: [PATCH 3/7] feat: Update clone logging --- pokeapi_ditto/commands/clone.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pokeapi_ditto/commands/clone.py b/pokeapi_ditto/commands/clone.py index a4bf22462..9eff6dd27 100644 --- a/pokeapi_ditto/commands/clone.py +++ b/pokeapi_ditto/commands/clone.py @@ -1,4 +1,5 @@ import os +import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Any, Callable, List, NamedTuple, Tuple @@ -43,6 +44,7 @@ def _calculate_max_workers() -> int: def _do_in_parallel( worker: Callable[[Tuple[str, str]], None], data: List[Tuple[str, str]], desc: str ) -> None: + t0 = time.monotonic() with ThreadPoolExecutor(max_workers=_MAX_WORKERS) as executor: futures = [executor.submit(worker, item) for item in data] try: @@ -58,6 +60,10 @@ def _do_in_parallel( except KeyboardInterrupt: executor.shutdown(wait=False, cancel_futures=True) raise + elapsed = time.monotonic() - t0 + tqdm.write( + f" done {desc:<30} {len(data):>5} resources {_MAX_WORKERS}T {elapsed:.1f}s" + ) class Cloner: @@ -163,7 +169,9 @@ def do_clone(src_url: str, dest_dir: str, select: List[str]) -> None: for sel in select: if "/" in sel: cloner.clone_single( - tuple(filter(None, sel.split("/")))[0:2] # pyright: ignore[reportArgumentType] + tuple(filter(None, sel.split("/")))[ + 0:2 + ] # pyright: ignore[reportArgumentType] ) else: cloner.clone_endpoint(sel) From 395ac31d1e39307a0e6d6a8c937d54d86bc8ba59 Mon Sep 17 00:00:00 2001 From: Naramsim Date: Fri, 21 Aug 2026 10:44:04 +0900 Subject: [PATCH 4/7] chore: bump version/trigger pipeline --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 643f54c69..ef0533882 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [tool.poetry] name = "pokeapi-ditto" -version = "1.0.4" +version = "1.0.5" description = "Ditto is a command line tool for performing meta operations over PokéAPI data." license = "Apache-2.0" -authors = ["Sargun Vohra "] +authors = ["Sargun Vohra ", "Triyan Mukherjee "] readme = "README.md" homepage = "https://github.com/PokeAPI/ditto" repository = "https://github.com/PokeAPI/ditto.git" From 6fbd4af6f6211cd354d143c3a2784ea9d6bf44ee Mon Sep 17 00:00:00 2001 From: Naramsim Date: Fri, 21 Aug 2026 11:18:05 +0900 Subject: [PATCH 5/7] refactor: use standard naming convention --- pokeapi_ditto/commands/transform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pokeapi_ditto/commands/transform.py b/pokeapi_ditto/commands/transform.py index 45b53ca88..e853794a9 100644 --- a/pokeapi_ditto/commands/transform.py +++ b/pokeapi_ditto/commands/transform.py @@ -27,9 +27,9 @@ def _dump(path: Path, content: Any): # clone.py is a cleaner model to follow -def do_transform(source_directory: str, destination_directory: str, base_url: str): - src_dir: Path = Path(source_directory) - dest_dir: Path = Path(destination_directory) +def do_transform(src_dir: str, dest_dir: str, base_url: str): + src_dir: Path = Path(src_dir) + dest_dir: Path = Path(dest_dir) if base_url.endswith("/"): base_url = base_url[:-1] From 3b5cf7eb6b80d553c69e956d8d47ca2e6e77af2d Mon Sep 17 00:00:00 2001 From: Triyan Mukherjee Date: Fri, 21 Aug 2026 19:42:15 +0530 Subject: [PATCH 6/7] chore: limit threads for now, hardfail on any exception --- pokeapi_ditto/commands/clone.py | 21 +++++++++------------ pokeapi_ditto/commands/transform.py | 12 ++++++------ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pokeapi_ditto/commands/clone.py b/pokeapi_ditto/commands/clone.py index 9eff6dd27..305ad3832 100644 --- a/pokeapi_ditto/commands/clone.py +++ b/pokeapi_ditto/commands/clone.py @@ -21,8 +21,7 @@ class RequestTimeout(NamedTuple): def _calculate_max_workers() -> int: - """ - Derive client thread count from co-located server capacity. + """Derive client thread count from co-located server capacity. https://github.com/PokeAPI/pokeapi/blob/master/gunicorn.conf.py @@ -30,12 +29,12 @@ def _calculate_max_workers() -> int: and the server uses gunicorn's default worker formula: 2 * cpu_count. We target 1.5x the server's worker count to keep the request pipeline saturated (accounting for network/IO round-trip slack) without starving - the server of CPU time. Capped at 24 to bound memory and fd usage. + the server of CPU time. """ cpu = os.cpu_count() or 4 server_workers = 2 * cpu # gunicorn default: 2 * CPU count client_threads = int(server_workers * 1.5) # 1.5x to fill the pipeline - return min(max(4, client_threads), 24) + return min(max(4, client_threads), 8) _MAX_WORKERS = _calculate_max_workers() @@ -57,7 +56,7 @@ def _do_in_parallel( leave=False, ): future.result() - except KeyboardInterrupt: + except BaseException: executor.shutdown(wait=False, cancel_futures=True) raise elapsed = time.monotonic() - t0 @@ -86,8 +85,8 @@ def __init__(self, src_url: str, dest_dir: str): def _build_session() -> requests.Session: session = requests.Session() retry = Retry( - total=3, - backoff_factor=0.5, + total=5, + backoff_factor=1.0, status_forcelist=[500, 502, 503, 504], allowed_methods=["GET"], ) @@ -106,11 +105,9 @@ def _crawl(self, url: URL, save: bool = True) -> Any: response.raise_for_status() data = orjson.loads(response.content) except requests.RequestException as e: - tqdm.write(f"Request failure: {url} ({e})") - return None - except orjson.JSONDecodeError: - tqdm.write(f"JSON decode failure: {url}") - return None + raise RuntimeError(f"Request failure: {url} ({e})") from e + except orjson.JSONDecodeError as e: + raise RuntimeError(f"JSON decode failure: {url} ({e})") from e if save: out_data = orjson.dumps(data, option=orjson.OPT_INDENT_2) diff --git a/pokeapi_ditto/commands/transform.py b/pokeapi_ditto/commands/transform.py index e853794a9..6fbac5911 100644 --- a/pokeapi_ditto/commands/transform.py +++ b/pokeapi_ditto/commands/transform.py @@ -28,16 +28,16 @@ def _dump(path: Path, content: Any): def do_transform(src_dir: str, dest_dir: str, base_url: str): - src_dir: Path = Path(src_dir) - dest_dir: Path = Path(dest_dir) + src_path = Path(src_dir) + dest_path = Path(dest_dir) if base_url.endswith("/"): base_url = base_url[:-1] - if not dest_dir.exists(): - dest_dir.mkdir(parents=True) + if not dest_path.exists(): + dest_path.mkdir(parents=True) - src_paths = src_dir.glob("**/*.json") + src_paths = src_path.glob("**/*.json") for src_path in tqdm(list(src_paths)): content: Dict[str, Any] = orjson.loads( @@ -45,7 +45,7 @@ def do_transform(src_dir: str, dest_dir: str, base_url: str): ) # all files - dest_path = dest_dir.joinpath(src_path.relative_to(src_dir)) + dest_path = dest_path.joinpath(src_path.relative_to(src_path)) _dump(dest_path, content) # named resource files From 8dc4c0fe890d38969f28245286ae480b462b1a52 Mon Sep 17 00:00:00 2001 From: Triyan Mukherjee Date: Fri, 21 Aug 2026 21:47:02 +0530 Subject: [PATCH 7/7] fix: fix variable shadowing in transform for fn parameter --- pokeapi_ditto/commands/transform.py | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pokeapi_ditto/commands/transform.py b/pokeapi_ditto/commands/transform.py index 6fbac5911..c030e9c3e 100644 --- a/pokeapi_ditto/commands/transform.py +++ b/pokeapi_ditto/commands/transform.py @@ -7,7 +7,7 @@ from pokeapi_ditto.common import apply_base_url -def _is_id(s: str): +def _is_id(s: str) -> bool: try: int(s) return True @@ -15,9 +15,9 @@ def _is_id(s: str): return False -def _dump(path: Path, content: Any): +def _dump(path: Path, content: Any) -> None: if not path.parent.exists(): - path.parent.mkdir(parents=True) + path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(orjson.dumps(content, option=orjson.OPT_INDENT_2)) @@ -27,41 +27,41 @@ def _dump(path: Path, content: Any): # clone.py is a cleaner model to follow -def do_transform(src_dir: str, dest_dir: str, base_url: str): - src_path = Path(src_dir) - dest_path = Path(dest_dir) +def do_transform(src_dir: str, dest_dir: str, base_url: str) -> None: + src_dir_path = Path(src_dir) + dest_dir_path = Path(dest_dir) if base_url.endswith("/"): base_url = base_url[:-1] - if not dest_path.exists(): - dest_path.mkdir(parents=True) + if not dest_dir_path.exists(): + dest_dir_path.mkdir(parents=True, exist_ok=True) - src_paths = src_path.glob("**/*.json") + src_files = list(src_dir_path.glob("**/*.json")) - for src_path in tqdm(list(src_paths)): + for file_path in tqdm(src_files): content: Dict[str, Any] = orjson.loads( - apply_base_url(src_path.read_text(), base_url) + apply_base_url(file_path.read_text(), base_url) ) # all files - dest_path = dest_path.joinpath(src_path.relative_to(src_path)) - _dump(dest_path, content) + dest_file = dest_dir_path.joinpath(file_path.relative_to(src_dir_path)) + _dump(dest_file, content) # named resource files - if _is_id(dest_path.parent.name) and "name" in content: + if _is_id(dest_file.parent.name) and "name" in content: name = content["name"] - dest_path = dest_path.parent.parent.joinpath(name, "index.json") - _dump(dest_path, content) + named_dest_file = dest_file.parent.parent.joinpath(name, "index.json") + _dump(named_dest_file, content) # a hack for pokemon/ID/encounters if ( - _is_id(dest_path.parent.parent.name) - and dest_path.parent.name == "encounters" + _is_id(dest_file.parent.parent.name) + and dest_file.parent.name == "encounters" ): - pokemon_path = src_path.parent.parent.joinpath("index.json") + pokemon_path = file_path.parent.parent.joinpath("index.json") name = orjson.loads(pokemon_path.read_bytes())["name"] - dest_path = dest_path.parent.parent.parent.joinpath( + enc_dest_file = dest_file.parent.parent.parent.joinpath( name, "encounters", "index.json" ) - _dump(dest_path, content) + _dump(enc_dest_file, content)