diff --git a/.deepsource.toml b/.deepsource.toml deleted file mode 100644 index 0ac63d3..0000000 --- a/.deepsource.toml +++ /dev/null @@ -1,7 +0,0 @@ -version = 1 - -[[analyzers]] -name = "python" - - [analyzers.meta] - runtime_version = "3.x.x" \ No newline at end of file diff --git a/README.md b/README.md index ab5ca76..6c44bb1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # thu-learn-downloader -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/505052ec12444da49201278a3abb2b7d)](https://app.codacy.com/gh/liblaf/thu-learn-downloader?utm_source=github.com&utm_medium=referral&utm_content=liblaf/thu-learn-downloader&utm_campaign=Badge_Grade) - Download everything from Web Learning of Tsinghua University ## Demo diff --git a/poetry.lock b/poetry.lock index b31f562..71a36a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand. [[package]] name = "altgraph" @@ -14,14 +14,14 @@ [[package]] name = "beautifulsoup4" -version = "4.12.0" +version = "4.12.2" description = "Screen-scraping library" category = "main" optional = false python-versions = ">=3.6.0" files = [ - {file = "beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"}, - {file = "beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"}, + {file = "beautifulsoup4-4.12.2-py3-none-any.whl", hash = "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"}, + {file = "beautifulsoup4-4.12.2.tar.gz", hash = "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da"}, ] [package.dependencies] @@ -278,14 +278,14 @@ [[package]] name = "packaging" -version = "23.0" +version = "23.1" description = "Core utilities for Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, ] [[package]] @@ -330,14 +330,14 @@ [[package]] name = "pygments" -version = "2.14.0" +version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "Pygments-2.14.0-py3-none-any.whl", hash = "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717"}, - {file = "Pygments-2.14.0.tar.gz", hash = "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297"}, + {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, + {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, ] [package.extras] @@ -345,24 +345,24 @@ [[package]] name = "pyinstaller" -version = "5.9.0" +version = "5.10.1" description = "PyInstaller bundles a Python application and all its dependencies into a single package." category = "dev" optional = false python-versions = "<3.12,>=3.7" files = [ - {file = "pyinstaller-5.9.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:93d7e8443a6b60745d42aa50f08730f6b419410832b4c616c4f1bb315f087661"}, - {file = "pyinstaller-5.9.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3b2c34c3c3ddf38f68d9f5afbed82abe0f89d53014c56892326fef10172ee652"}, - {file = "pyinstaller-5.9.0-py3-none-manylinux2014_i686.whl", hash = "sha256:dcd348b174fd72c4df271790ac582969c9423cb099fe92db9ec131a8a9243d5a"}, - {file = "pyinstaller-5.9.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:4b21b0298db44f5f07fc04d8ff81ec31efa47b72798efaecc4e811c50a102111"}, - {file = "pyinstaller-5.9.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:12ca6567be457826e14416637ea54485a185d0ce7a5a044df0d0daf588fff6d1"}, - {file = "pyinstaller-5.9.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:c7dd156c2438f197c168b990bbce03c97d3fb758dd9bbc3ca93626c2f4473a47"}, - {file = "pyinstaller-5.9.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:2ba42038b3bd83e1fba7c8eb9e7cde43bd5938e37ca542c89e8779355d213f52"}, - {file = "pyinstaller-5.9.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:d1ff94347183ae3755cfb8f02e64744eb7fe384469bd61e453c6ff59a81665d6"}, - {file = "pyinstaller-5.9.0-py3-none-win32.whl", hash = "sha256:8476538aec8a0a3be4f74b93388bd6989b91cc437ff86d6f0d3a68961176dce6"}, - {file = "pyinstaller-5.9.0-py3-none-win_amd64.whl", hash = "sha256:e7a4c292810285c2466f3bdcb1e03ba2170177ebe3d7054ff1af3bb348bf61a4"}, - {file = "pyinstaller-5.9.0-py3-none-win_arm64.whl", hash = "sha256:6cf6c032c72ef78fd9aa5e47d8952e784db45b2c3f7862bd44a99df68c216f64"}, - {file = "pyinstaller-5.9.0.tar.gz", hash = "sha256:2bde16a8d664e8eba9aa7b84f729f7ab005c1793be4fe1986b3c9cad6c486622"}, + {file = "pyinstaller-5.10.1-py3-none-macosx_10_13_universal2.whl", hash = "sha256:247b99c52dc3cf69eba905da30dbca0a8ea309e1058cab44658ac838d9b8f2f0"}, + {file = "pyinstaller-5.10.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:2d16641a495593d174504263b038a6d3d46b3b15a381ccb216cf6cce67723512"}, + {file = "pyinstaller-5.10.1-py3-none-manylinux2014_i686.whl", hash = "sha256:df97aaf1103a1c485aa3c9947792a86675e370f5ce9b436b4a84e34a4180c8d2"}, + {file = "pyinstaller-5.10.1-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:333b4ffda38d9c0a561c38429dd9848d37aa78f3b8ea8a6f2b2e69a60d523c02"}, + {file = "pyinstaller-5.10.1-py3-none-manylinux2014_s390x.whl", hash = "sha256:6afc7aa4885ffd3e6121a8cf2138830099f874c18cb5869bed8c1a42db82d060"}, + {file = "pyinstaller-5.10.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:85e39e36d03355423636907a26a9bfa06fdc93cb1086441b19d2d0ca448479fa"}, + {file = "pyinstaller-5.10.1-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:7a1db833bb0302b66ae3ae337fbd5487699658ce869ca4d538b5359b8179e83a"}, + {file = "pyinstaller-5.10.1-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:bb7de35cd209a0a0358aec761a273ae951d2161c03728f15d9a640d06a88e472"}, + {file = "pyinstaller-5.10.1-py3-none-win32.whl", hash = "sha256:9e9a38f41f8280c8e29b294716992852281b41fbe64ba330ebab671efe27b26d"}, + {file = "pyinstaller-5.10.1-py3-none-win_amd64.whl", hash = "sha256:915a502802c751bafd92d568ac57468ec6cdf252b8308aa9a167bbc2c565ad2d"}, + {file = "pyinstaller-5.10.1-py3-none-win_arm64.whl", hash = "sha256:f677fbc151db1eb00ada94e86ed128e7b359cbd6bf3f6ea815afdde687692d46"}, + {file = "pyinstaller-5.10.1.tar.gz", hash = "sha256:6ecc464bf56919bf2d6bff275f38d85ff08ae747b8ead3a0c26cf85573b3c723"}, ] [package.dependencies] @@ -379,14 +379,14 @@ [[package]] name = "pyinstaller-hooks-contrib" -version = "2023.1" +version = "2023.2" description = "Community maintained hooks for PyInstaller" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "pyinstaller-hooks-contrib-2023.1.tar.gz", hash = "sha256:ab56c192e7cd4472ff6b840cda4fc42bceccc7fb4234f064fc834a3248c0afdd"}, - {file = "pyinstaller_hooks_contrib-2023.1-py2.py3-none-any.whl", hash = "sha256:d2ea40a7105651aa525bfe5fe309aa264d4d9bb49f839b862243dcf0a56c34cd"}, + {file = "pyinstaller-hooks-contrib-2023.2.tar.gz", hash = "sha256:7fb856a81fd06a717188a3175caa77e902035cc067b00b583c6409c62497b23f"}, + {file = "pyinstaller_hooks_contrib-2023.2-py2.py3-none-any.whl", hash = "sha256:e02c5f0ee3d4f5814588c2128caf5036c058ba764aaf24d957bb5311ad8690ad"}, ] [[package]] @@ -444,14 +444,14 @@ [[package]] name = "setuptools" -version = "67.6.0" +version = "67.6.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, - {file = "setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, + {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, + {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, ] [package.extras] @@ -473,14 +473,14 @@ [[package]] name = "soupsieve" -version = "2.4" +version = "2.4.1" description = "A modern CSS selector implementation for Beautiful Soup." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "soupsieve-2.4-py3-none-any.whl", hash = "sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955"}, - {file = "soupsieve-2.4.tar.gz", hash = "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a"}, + {file = "soupsieve-2.4.1-py3-none-any.whl", hash = "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8"}, + {file = "soupsieve-2.4.1.tar.gz", hash = "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"}, ] [[package]] diff --git a/requirements.txt b/requirements.txt index fdf23a4..c5e1a90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ -beautifulsoup4==4.12.0 ; python_version >= "3.10" and python_version < "3.12" +beautifulsoup4==4.12.2 ; python_version >= "3.10" and python_version < "3.12" certifi==2022.12.7 ; python_version >= "3.10" and python_version < "3.12" charset-normalizer==3.1.0 ; python_version >= "3.10" and python_version < "3.12" click==8.1.3 ; python_version >= "3.10" and python_version < "3.12" colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.12" commonmark==0.9.1 ; python_version >= "3.10" and python_version < "3.12" idna==3.4 ; python_version >= "3.10" and python_version < "3.12" -pygments==2.14.0 ; python_version >= "3.10" and python_version < "3.12" +pygments==2.15.1 ; python_version >= "3.10" and python_version < "3.12" requests==2.28.2 ; python_version >= "3.10" and python_version < "3.12" rich==12.6.0 ; python_version >= "3.10" and python_version < "3.12" shellingham==1.5.0.post1 ; python_version >= "3.10" and python_version < "3.12" -soupsieve==2.4 ; python_version >= "3.10" and python_version < "3.12" +soupsieve==2.4.1 ; python_version >= "3.10" and python_version < "3.12" typer[all]==0.7.0 ; python_version >= "3.10" and python_version < "3.12" urllib3==1.26.15 ; python_version >= "3.10" and python_version < "3.12" diff --git a/thu_learn_downloader/parser.py b/thu_learn_downloader/parser.py index a0d00f3..9e895fd 100644 --- a/thu_learn_downloader/parser.py +++ b/thu_learn_downloader/parser.py @@ -105,9 +105,10 @@ def parse_homework_file(file_div: Tag) -> Optional[t.RemoteFile]: - ftitle = file_div.select_one(".ftitle") or file_div.select_one(".fl") - assert ftitle - file_node = ftitle.select_one("a") + fl = file_div.select_one(".txt.fl") + if not fl: + return None + file_node = fl.select_one("a") if not file_node: return None diff --git a/thu_learn_downloader/utils.py b/thu_learn_downloader/utils.py index b7a0a61..7d5c0c6 100644 --- a/thu_learn_downloader/utils.py +++ b/thu_learn_downloader/utils.py @@ -40,7 +40,7 @@ def remove_attachment_prefix(name: str) -> str: - prefixes: list[str] = ["attach", "ans", "submit", "comment", "-"] + prefixes: list[str] = ["attach-", "ans-", "submit-", "comment-", "-"] while name.startswith(tuple(prefixes)): for p in prefixes: name = name.removeprefix(p)