[tmp] Build AppImage #1519
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| on: | |
| push: | |
| branches: '**' | |
| tags-ignore: '**' | |
| pull_request: | |
| jobs: | |
| Static-Code-Checks: | |
| runs-on: ubuntu-latest | |
| env: | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Pytype does not support Python 3.13 as of 2025-06. | |
| python-version: '3.12' | |
| - name: Install pip Dependencies | |
| run: | | |
| sudo apt-get -y update | |
| sudo apt-get -y install liblzo2-dev | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --user pytest lz4 PySquashfsImage asyncssh fsspec pygit2 sqlcipher3-wheels | |
| ( cd core && python3 -m pip install --user .[full,ext4,fsspec,sqlar] ) | |
| python3 -m pip install --force-reinstall 'git+https://github.com/mxmlnkn/mfusepy.git@master#egginfo=mfusepy' | |
| python3 -m pip install --user . | |
| - name: Style Check With Ruff | |
| run: | | |
| python3 -m pip install ruff | |
| ruff check --config tests/.ruff.toml -- $( git ls-tree -r --name-only HEAD | 'grep' -E '[.]py$' | 'grep' -v '/_external/' ) | |
| - name: Style Check With Black | |
| run: | | |
| python3 -m pip install black | |
| black -q --diff --line-length 120 --skip-string-normalization $( git ls-tree -r --name-only HEAD | 'grep' '[.]py$' | 'grep' -v '/_external/' ) > black.diff | |
| if [ -s black.diff ]; then | |
| cat black.diff | |
| exit 123 | |
| fi | |
| - name: Lint With Codespell | |
| run: | | |
| python3 -m pip install codespell | |
| codespell $( git ls-tree -r --name-only HEAD | 'grep' -E '[.](py|md|txt|sh|yml)$' | 'grep' -v '/_external/' ) | |
| - name: Lint With Flake8 | |
| run: | | |
| python3 -m pip install flake8 | |
| flake8 --config tests/.flake8 {ratarmount,tests}/[^_]*.py core/{ratarmountcore,tests}/[^_]*.py | |
| - name: Lint With Pylint | |
| run: | | |
| python3 -m pip install pylint | |
| pylint --rcfile tests/.pylintrc ratarmount core/ratarmountcore tests/*.py core/tests/*.py | tee pylint.log | |
| ! 'egrep' ': E[0-9]{4}: ' pylint.log | |
| - name: Lint With Mypy | |
| run: | | |
| yes | python3 -m pip install --upgrade-strategy eager --upgrade types-dataclasses mypy | |
| mypy --config-file tests/.mypy.ini ratarmount core/ratarmountcore core/tests | |
| yes | python3 -m pip uninstall types-dataclasses | |
| - name: Lint With ShellCheck | |
| run: | | |
| sudo apt-get -y install shellcheck | |
| shellcheck -e SC2064 tests/*.sh | |
| Longer-Running-Checks: | |
| runs-on: ubuntu-latest | |
| needs: [Static-Code-Checks] | |
| env: | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| # Pytype does not support Python 3.13 as of 2025-06. | |
| python-version: '3.12' | |
| - name: Install pip Dependencies | |
| run: | | |
| sudo apt-get -y update | |
| sudo apt-get -y install liblzo2-dev | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --user pytest lz4 PySquashfsImage asyncssh fsspec pygit2 sqlcipher3-wheels | |
| ( cd core && python3 -m pip install --user .[full,ext4,fsspec,sqlar] ) | |
| python3 -m pip install --force-reinstall 'git+https://github.com/mxmlnkn/mfusepy.git@master#egginfo=mfusepy' | |
| python3 -m pip install --user . | |
| - name: Lint With Pytype | |
| run: | | |
| python3 -m pip install pytype | |
| pytype -j auto -d import-error -P$( cd core && pwd ):$( pwd ) --exclude=core/ratarmountcore/_external ratarmount core/ratarmountcore core/tests | |
| Tests: | |
| runs-on: ${{ matrix.os }} | |
| needs: [Static-Code-Checks] | |
| strategy: | |
| # Disable because it makes no sense. Most often, they run in parallel, i.e., they would fail in roughly the | |
| # same time anyway if the error persists on every system, and if not, it would be nice to know which systems | |
| # work and which don't. If multiple systems are affected it might only extend debugging because I would only | |
| # notice the error on the next CI run. | |
| fail-fast: false | |
| matrix: | |
| os: ['macos-13', 'ubuntu-latest'] | |
| # macos-13 / macos-latest does not work anymore because the dependencies don't have any wheels, | |
| # probably because it is M1 based. | |
| # ToDo: Add windows-latest but it requires a lot of setup of the dependencies! | |
| # Maybe only test ratarmount-core without most dependencies after I have split that off. | |
| # Oldest and newest versions should be enough. Python versions are supported 5 years from release date. | |
| # https://endoflife.date/python | |
| # 3.5 was released 2015-09-13 and end-of-life was 2020-09-13 | |
| # 3.6 was released 2016-12-23 and end-of-life was 2021-12-23 | |
| # 3.7 was released 2018-06-27 and end-of-life was 2023-06-27 | |
| # 3.8 was released 2019-10-14 and end-of-life was 2024-10-14 | |
| # 3.9 was released 2020-10-05 and end-of-life will be 2025-10-25 | |
| # 3.10 was released 2021-10-04 and end-of-life will be 2026-10-04 | |
| # 3.11 was released 2022-10-24 and end-of-life will be 2027-10 | |
| # 3.12 was released 2023-10-02 and end-of-life will be 2028-10 | |
| # 3.13 was released 2024-10-07 and end-of-life will be 2029-10 | |
| # 3.14 is to be released 2025-10 | |
| # Supported versions: https://github.com/actions/python-versions/blob/main/versions-manifest.json | |
| python-version: ['3.9', '3.12', '3.13'] | |
| libfuse-version: ['system'] | |
| include: | |
| - os: ubuntu-latest | |
| python-version: '3.14.0-beta.4' | |
| libfuse-version: 'system' | |
| - os: ubuntu-latest | |
| python-version: '3.13' | |
| libfuse-version: '3.17.2' | |
| defaults: | |
| run: | |
| # This is especially important for windows because it seems to default to powershell | |
| shell: bash | |
| env: | |
| LIBFUSE_VERSION: ${{ matrix.libfuse-version }} | |
| RATARMOUNT_CHECK_OVERRIDES: 1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| # We need one tag for testing the git mount. | |
| # This is BROKEN! God damn it. Is anything working at all... | |
| # https://github.com/actions/checkout/issues/1781 | |
| fetch-tags: true | |
| - name: Fetch tag for tests | |
| run: git fetch origin refs/tags/v0.15.2:refs/tags/v0.15.2 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Print System Information | |
| run: | | |
| echo "uname -a: $( uname -a )" | |
| echo "Shell: $SHELL" | |
| echo "Cores: $( nproc )" | |
| echo "Mount points:"; mount | |
| echo "/etc/fuse.conf:"; cat /etc/fuse.conf || true | |
| - uses: msys2/setup-msys2@v2 | |
| if: startsWith( matrix.os, 'windows' ) | |
| with: | |
| install: gcc make liblzma-devel libzstd-devel zlib-devel | |
| - name: Install Dependencies (Linux) | |
| if: startsWith( matrix.os, 'ubuntu' ) | |
| run: | | |
| # Libarchive calls the grzip, lrzip, lzop binaries for lrzip support. Others, such as bzip2, gzip, lz4, lzma, | |
| # zstd, may also call external binaries depending on how libarchive was compiled! | |
| # https://github.com/libarchive/libarchive/blob/ad5a0b542c027883d7069f6844045e6788c7d70c/libarchive/ | |
| # archive_read_support_filter_lrzip.c#L68 | |
| sudo apt-get -y update | |
| sudo apt-get -y install libfuse2 fuse3 bzip2 pbzip2 pixz zstd unar lrzip lzop gcc liblzo2-dev ruby-webrick | |
| sudo apt-get -y install liblzma-dev* | |
| - name: Install Dependencies For Unreleased Python Versions (Linux) | |
| if: startsWith( matrix.os, 'ubuntu' ) && startsWith( matrix.python-version, '3.14.0-' ) | |
| run: | | |
| #libgit2-dev is too old on Ubuntu 22.04. Leads to error about missing git2/sys/errors.h | |
| #sudo apt-get -y install libgit2-dev | |
| sudo apt-get -y install cmake | |
| git clone --branch v1.8.1 --depth 1 https://github.com/libgit2/libgit2.git | |
| ( cd libgit2 && mkdir build && cd build && cmake .. && cmake --build . && sudo cmake --build . -- install ) | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib" >> "$GITHUB_ENV" | |
| # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies. | |
| python3 -m pip install pygit2==1.15 | |
| - name: Install Dependencies (MacOS) | |
| if: startsWith( matrix.os, 'macos' ) | |
| run: | | |
| # coreutils is required for the tests written in shell, e.g., for the 'realpath' command | |
| # unar is required for rar tests with passwords. By default, bsdtar is installed but that is the only | |
| # one of the three supported tools (the third is unrar) that does not support passwords. | |
| # And the error message is atrocious: | |
| # cmdline.extend(args) | |
| # TypeError: 'NoneType' object is not iterable | |
| brew install -q macfuse coreutils pixz pbzip2 zstd unar libarchive lrzip lzop lzo | |
| # Add brew installation binary folder to PATH so that command line tools like zstd can be found | |
| echo PATH="$PATH:/usr/local/bin" >> "$GITHUB_ENV" | |
| - name: Install Dependencies For Unreleased Python Versions (MacOS) | |
| if: > | |
| startsWith( matrix.os, 'macos' ) && ( | |
| matrix.python-version == '3.13' || | |
| startsWith( matrix.python-version, '3.14' ) ) | |
| run: | | |
| brew install -q [email protected] | |
| brew link [email protected] --force | |
| # Expects exactly libgit2 1.8.x! Therefore install it here instead via the non-pinned ratarmount dependencies. | |
| python3 -m pip install pygit2==1.15 | |
| - name: Install pip Dependencies | |
| run: | | |
| python3 -m pip install --upgrade pip | |
| # Still needed for building lzmaffi without build isolation | |
| python3 -m pip install --upgrade wheel | |
| python3 -m pip install --upgrade setuptools | |
| python3 -m pip install --upgrade-strategy eager --upgrade twine build cffi | |
| - name: Install Custom libFUSE | |
| if: matrix.libfuse-version != 'system' | |
| run: | | |
| NAME=fuse-${LIBFUSE_VERSION} | |
| wget https://github.com/libfuse/libfuse/releases/download/${NAME}/${NAME}.tar.gz | |
| tar -xf "${NAME}.tar.gz" | |
| cd -- "${NAME}" | |
| mkdir -p build | |
| cd build | |
| sudo apt-get -y install meson python*-pytest | |
| meson .. | |
| ninja | |
| #sudo python3 -m pytest test/ | |
| sudo ninja install | |
| sudo ln -s /usr/local/bin/fusermount{3,} | |
| cd ../.. | |
| rm -rf -- "${NAME}" | |
| - name: Test ratarmountcore Installation From Tarball | |
| working-directory: core | |
| run: | | |
| python3 -m build | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full,ext4,fsspec,sqlar] | |
| python3 -m pip install --force-reinstall 'git+https://github.com/mxmlnkn/mfusepy.git@master#egginfo=mfusepy' | |
| - name: Test Startup Without Compression Dependencies | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| python3 -m ratarmount tests/simple.bz2 || [ $? != 139 ] | |
| - name: Test Startup With Only One Compression Dependency | |
| run: | | |
| for module in indexed_gzip indexed_zstd lzmaffi python-xz; do | |
| if [[ $module == lzmaffi ]]; then | |
| # --use-pep517 to fix deprecation warning: https://github.com/pypa/pip/issues/6334 | |
| python3 -m pip install --use-pep517 --no-build-isolation --upgrade "$module" | |
| elif [[ $module == indexed_gzip ]]; then | |
| python3 -m pip install --upgrade "indexed_gzip >= 1.6.3, < 1.9.3" | |
| fi | |
| # Segfaults (139) are not allowed but other exit codes are valid! | |
| # indexed_zstd=1.2.0 did segfault here! | |
| python3 -m ratarmount README.md || [ $? != 139 ] | |
| python3 -m pip uninstall --yes "$module" | |
| done | |
| python3 -m pip install --upgrade 'git+https://github.com/mxmlnkn/indexed_bzip2.git@master#egginfo=rapidgzip&subdirectory=python/rapidgzip' | |
| - name: Test Installation From Tarball | |
| run: | | |
| python3 -m build | |
| # Around commit 08ddc8fb3679faacd6e54eb5c54104995b045dc4, I had some very weird twine check failures | |
| # because .tar.gz tarballs were wrongly recognized as ZIP files by CPython's zipfile.is_zipfile. | |
| # Therefore, try recompressing the tarballs with different compression levels until it does not randomly | |
| # look like a ZIP anymore. It is an ugly hack, but so is zipfile.is_zipfile. | |
| tarball=$( find dist -name '*.tar.gz' ) | |
| if python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball"; then | |
| gzip -c -d "$tarball" > "$tarball.tar" | |
| for (( i=9; i>0; --i )); do | |
| cat "$tarball.tar" | gzip -$i > "$tarball" | |
| if ! python3 -c 'import sys, zipfile; sys.exit(0 if zipfile.is_zipfile(sys.argv[1]) else 1)' "$tarball" | |
| then break; fi | |
| done | |
| fi | |
| twine check dist/* | |
| python3 -m pip install "$( find dist -name '*.tar.gz' | head -1 )"[full] | |
| - name: Test Installation From Source | |
| run: | | |
| python3 -m pip install .[full,ext4,fsspec,sqlar] | |
| python3 -m pip install --force-reinstall 'git+https://github.com/mxmlnkn/mfusepy.git@master#egginfo=mfusepy' | |
| - name: Test Simple Startup | |
| run: | | |
| ratarmount --help | |
| ratarmount --version | |
| - name: Test Simple Mount | |
| # macOS 11+ is too uptight about "security" and is not able to fully load the macfuse kernel extension. | |
| # https://github.com/actions/runner-images/issues/4731 | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| ratarmount tests/single-file.tar mimi | |
| ls -la mimi | |
| sleep 1s | |
| # MacOS does not have fusermount! | |
| ratarmount -u mimi | |
| - name: CLI Tests (Linux) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| python3 -m pip install pytest | |
| # First off, n=auto seems to use the physical cores and ignores virtual ones. | |
| # Secondly, these tests scale much better than the others because most time is spent waiting for | |
| # the FUSE mount point to appear or disappear, which doesn't seem to be bottlenecked by CPU usage. | |
| # There is a new problem in the Github Actions CI: | |
| # fusermount: too many FUSE filesystems mounted; mount_max=N can be set in /etc/fuse.conf | |
| # The output of 'mount' shows me only 4 ratarmount mounts! | |
| # And now, with 4 mounts, it simply fails without an error message ... WTF is going on. | |
| # Remove any multi-threaded execution seems to work, but the error occurred sporadically, so maybe it's that. | |
| python3 -X dev -W ignore::DeprecationWarning:fuse -u \ | |
| -c "import pytest, sys; sys.exit(pytest.console_main())" \ | |
| --disable-warnings tests/test_cli.py | |
| - name: Unit Tests | |
| run: | | |
| python3 -m pip install pytest pytest-xdist pandas | |
| for file in core/tests/test_*.py; do | |
| case "$file" in | |
| "core/tests/test_AutoMountLayer.py"\ | |
| |"core/tests/test_BlockParallelReaders.py"\ | |
| |"core/tests/test_LibarchiveMountSource.py"\ | |
| |"core/tests/test_SQLiteIndexedTar.py") | |
| echo "$file" # pytest-xdist seems to omit the test file name | |
| pytest -n auto --disable-warnings "$file" | |
| ;; | |
| *) | |
| # Fusepy warns about usage of use_ns because the implicit behavior is deprecated. | |
| # But there has been no development to fusepy for 4 years, so I think it should be fine to ignore. | |
| pytest --disable-warnings "$file" | |
| esac | |
| done | |
| python3 tests/tests.py | |
| - name: Install pip Test Dependencies | |
| run: | | |
| python3 -m pip install -r tests/requirements-tests.txt | |
| - name: Install Test Dependencies (Linux) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) }} | |
| run: | | |
| sudo bash tests/install-smbd.sh | |
| - name: Regression Tests (FUSE 3) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) && matrix.libfuse-version != 'system' }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| run: | | |
| if [[ $LIBFUSE_VERSION == system ]]; then | |
| export FUSE_LIBRARY_PATH=$( dpkg -L libfuse3-3 | 'grep' -F .so | head -1 ) | |
| else | |
| export FUSE_LIBRARY_PATH=/usr/local/lib/x86_64-linux-gnu/libfuse3.so.$LIBFUSE_VERSION | |
| fi | |
| ratarmount --version | grep -i fuse | |
| bash tests/runtests.sh | |
| - name: Regression Tests (FUSE 2) | |
| if: ${{ !startsWith( matrix.os, 'macos' ) && matrix.libfuse-version == 'system' }} | |
| env: | |
| DROPBOX_TOKEN: ${{ secrets.DROPBOX_TOKEN }} | |
| run: | | |
| ratarmount --version | grep -i fuse | |
| bash tests/runtests.sh | |
| - name: Module tests without fusepy | |
| run: | | |
| python3 -m pip uninstall -y fusepy mfusepy | |
| python3 tests/tests.py |