Ignore more artifacts

.github/workflows/ci.yml

@@ -79,7 +79,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
 
     - name: Install Python dependencies
       run: |
@@ -200,7 +200,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
         sudo apt-get install -y bash zsh fish  # for shell completion tests
         sudo apt-get install -y rclone openssh-server curl
         if [[ "$TOXENV" == *"llfuse"* ]]; then
@@ -435,7 +435,7 @@ jobs:
               freebsd)
                 export IGNORE_OSVERSION=yes
                 sudo -E pkg update -f
-                sudo -E pkg install -y xxhash liblz4 zstd pkgconf
+                sudo -E pkg install -y xxhash liblz4 pkgconf
                 sudo -E pkg install -y fusefs-libs
                 sudo -E kldload fusefs
                 sudo -E sysctl vfs.usermount=1
@@ -491,7 +491,7 @@ jobs:
                 echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
                 sudo -E pkgin update
                 sudo -E pkgin -y upgrade
-                sudo -E pkgin -y install zstd lz4 xxhash git
+                sudo -E pkgin -y install lz4 xxhash git
                 sudo -E pkgin -y install rust
                 sudo -E pkgin -y install pkg-config
                 sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox
@@ -525,7 +525,7 @@ jobs:
                 ;;
 
               openbsd)
-                sudo -E pkg_add xxhash lz4 zstd git
+                sudo -E pkg_add xxhash lz4 git
                 sudo -E pkg_add rust
                 sudo -E pkg_add openssl%3.4
                 sudo -E pkg_add py3-pip py3-virtualenv py3-tox
@@ -563,12 +563,12 @@ jobs:
 
               haiku)
                 pkgman refresh
-                pkgman install -y git pkgconfig zstd lz4 xxhash
+                pkgman install -y git pkgconfig lz4 xxhash
                 pkgman install -y openssl3
                 pkgman install -y rust_bin
                 pkgman install -y python3.10
                 pkgman install -y cffi
-                pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel
+                pkgman install -y lz4_devel xxhash_devel openssl3_devel libffi_devel
 
                 # there is no pkgman package for tox, so we install it into a venv
                 python3 -m ensurepip --upgrade
@@ -578,7 +578,6 @@ jobs:
 
                 export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
                 export BORG_LIBLZ4_PREFIX=/system/develop
-                export BORG_LIBZSTD_PREFIX=/system/develop
                 export BORG_LIBXXHASH_PREFIX=/system/develop
                 export BORG_OPENSSL_PREFIX=/system/develop
                 pip install -r requirements.d/development.txt

.github/workflows/codeql-analysis.yml

@@ -66,7 +66,7 @@ jobs:
       run: |
         sudo apt-get update
         sudo apt-get install -y pkg-config build-essential
-        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
+        sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v4

.gitignore

@@ -28,3 +28,6 @@ borg.exe
 .coverage
 .coverage.*
 .vagrant
+src/borg/hashindex.c
+src/borg/platform/netbsd.c
+

.readthedocs.yaml

@@ -16,7 +16,6 @@ build:
         - libacl1-dev
         - libssl-dev
         - liblz4-dev
-        - libzstd-dev
         - libxxhash-dev
 
 python:

Brewfile

@@ -1,5 +1,4 @@
 brew 'pkgconf'
-brew 'zstd'
 brew 'lz4'
 brew 'xxhash'
 brew 'openssl@3'

Vagrantfile

@@ -16,7 +16,7 @@ def packages_debianoid(user)
     apt-get -y -qq dist-upgrade
     # for building borgbackup and dependencies:
     apt install -y pkg-config
-    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true
+    apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev || true
     apt install -y libfuse-dev fuse || true
     apt install -y libfuse3-dev fuse3 || true
     apt install -y locales || true
@@ -38,7 +38,7 @@ def packages_freebsd
     # install all the (security and other) updates, base system
     freebsd-update --not-running-from-cron fetch install
     # for building borgbackup and dependencies:
-    pkg install -y xxhash liblz4 zstd pkgconf
+    pkg install -y xxhash liblz4 pkgconf
     pkg install -y fusefs-libs || true
     pkg install -y fusefs-libs3 || true
     pkg install -y rust
@@ -85,7 +85,6 @@ def packages_openbsd
     chsh -s bash vagrant
     pkg_add xxhash
     pkg_add lz4
-    pkg_add zstd
     pkg_add git  # no fakeroot
     pkg_add rust
     pkg_add openssl%3.4
@@ -100,7 +99,7 @@ def packages_netbsd
     echo 'https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf
     pkgin update
     pkgin -y upgrade
-    pkg_add zstd lz4 xxhash git
+    pkg_add lz4 xxhash git
     pkg_add rust
     pkg_add bash
     chsh -s bash vagrant

docs/installation.rst

@@ -166,7 +166,6 @@ development header files (sometimes in a separate `-dev` or `-devel` package).
 * libacl_ (which depends on libattr_)
 * libxxhash_ >= 0.8.1
 * liblz4_ >= 1.7.0 (r129)
-* libzstd_ >= 1.3.0
 * libffi (required for argon2-cffi-bindings)
 * pkg-config (cli tool) - Borg uses this to discover header and library
   locations automatically. Alternatively, you can also point to them via some
@@ -201,7 +200,7 @@ Arch Linux
 
 Install the runtime and build dependencies::
 
-    pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 zstd base-devel
+    pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 base-devel
     pacman -S fuse2     # needed for llfuse
     pacman -S fuse3     # needed for pyfuse3
 
@@ -217,7 +216,7 @@ Install the dependencies with development headers::
     sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \
     libacl1-dev \
     libssl-dev \
-    liblz4-dev libzstd-dev libxxhash-dev \
+    liblz4-dev libxxhash-dev \
     libffi-dev \
     build-essential pkg-config
     sudo apt-get install libfuse-dev fuse    # needed for llfuse
@@ -235,7 +234,7 @@ Install the dependencies with development headers::
     sudo dnf install python3 python3-devel python3-pip python3-virtualenv \
     libacl-devel \
     openssl-devel \
-    lz4-devel libzstd-devel xxhash-devel \
+    lz4-devel xxhash-devel \
     libffi-devel \
     pkgconf
     sudo dnf install gcc gcc-c++ redhat-rpm-config
@@ -252,7 +251,7 @@ Install the dependencies automatically using zypper::
 Alternatively, you can enumerate all build dependencies in the command line::
 
     sudo zypper install python3 python3-devel \
-    libacl-devel openssl-devel xxhash-devel libzstd-devel liblz4-devel \
+    libacl-devel openssl-devel xxhash-devel liblz4-devel \
     libffi-devel \
     python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \
     python3-pytest python3-setuptools python3-setuptools_scm \
@@ -303,7 +302,7 @@ and commands to make FUSE work for using the mount command.
 
      pkg install -y python3 pkgconf
      pkg install openssl
-     pkg install liblz4 zstd xxhash
+     pkg install liblz4 xxhash
      pkg install fusefs-libs  # needed for llfuse
      pkg install -y git
      python3 -m ensurepip # to install pip for Python3
@@ -347,7 +346,7 @@ Use the Cygwin installer to install the dependencies::
 
     python39 python39-devel
     python39-setuptools python39-pip python39-wheel python39-virtualenv
-    libssl-devel libxxhash-devel liblz4-devel libzstd-devel
+    libssl-devel libxxhash-devel liblz4-devel
     binutils gcc-g++ git make openssh
 
 Make sure to use a virtual environment to avoid confusions with any Python installed on Windows.

docs/usage/general/environment.rst.inc

@@ -218,9 +218,6 @@ Building:
     BORG_LIBLZ4_PREFIX
         Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg
         will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
-    BORG_LIBZSTD_PREFIX
-        Adds given prefix directory to the default locations. If a 'include/zstd.h' is found Borg
-        will be linked against the system libzstd instead of a bundled implementation. (setup.py)
 
 Please note:
 

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
   "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'",  # for others: 2.6+ works consistently.
   "argon2-cffi",
   "shtab>=1.8.0",
+  "backports-zstd; python_version < '3.14'", # for python < 3.14.
 ]
 
 [project.optional-dependencies]

requirements.d/development.lock.txt

@@ -1,3 +1,4 @@
+chardet==5.2.0
 setuptools==80.9.0
 setuptools-scm==9.2.2
 pip==26.0.1

requirements.d/development.txt

@@ -1,3 +1,4 @@
+chardet < 6
 setuptools >=78.1.1
 setuptools_scm
 pip !=24.2

scripts/Dockerfile.linux-run

@@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libacl1-dev \
     libxxhash-dev \
     liblz4-dev \
-    libzstd-dev \
     libfuse3-dev \
     fuse3 \
     python3-dev \

scripts/msys2-install-deps

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
+pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
 
 if [ "$1" = "development" ]; then
 	pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

setup.py

@@ -160,7 +160,6 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
     compress_ext_kwargs = members_appended(
         dict(sources=[compress_source]),
         lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
-        lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"),
         dict(extra_compile_args=cflags),
     )
 

src/borg/archive.py

@@ -1242,7 +1242,7 @@ def __init__(
         log_json,
         iec,
         file_status_printer=None,
-        files_changed="ctime",
+        files_changed="mtime" if is_win32 else "ctime",
     ):
         self.metadata_collector = metadata_collector
         self.cache = cache
@@ -1471,40 +1471,35 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal,
                             )
                             self.stats.chunking_time = self.chunker.chunking_time
                         end_reading = time.time_ns()
-                        if not is_win32:  # TODO for win32
-                            with backup_io("fstat2"):
-                                st2 = os.fstat(fd)
-                            if self.files_changed == "disabled" or is_special_file:
-                                # special files:
-                                # - fifos change naturally, because they are fed from the other side. no problem.
-                                # - blk/chr devices don't change ctime anyway.
-                                pass
-                            elif self.files_changed == "ctime":
-                                if st.st_ctime_ns != st2.st_ctime_ns:
-                                    # ctime was changed, this is either a metadata or a data change.
-                                    changed_while_backup = True
-                                elif (
-                                    start_reading - TIME_DIFFERS1_NS < st2.st_ctime_ns < end_reading + TIME_DIFFERS1_NS
-                                ):
-                                    # this is to treat a very special race condition, see #3536.
-                                    # - file was changed right before st.ctime was determined.
-                                    # - then, shortly afterwards, but already while we read the file, the
-                                    #   file was changed again, but st2.ctime is the same due to ctime granularity.
-                                    # when comparing file ctime to local clock, widen interval by TIME_DIFFERS1_NS.
-                                    changed_while_backup = True
-                            elif self.files_changed == "mtime":
-                                if st.st_mtime_ns != st2.st_mtime_ns:
-                                    # mtime was changed, this is either a data change.
-                                    changed_while_backup = True
-                                elif (
-                                    start_reading - TIME_DIFFERS1_NS < st2.st_mtime_ns < end_reading + TIME_DIFFERS1_NS
-                                ):
-                                    # this is to treat a very special race condition, see #3536.
-                                    # - file was changed right before st.mtime was determined.
-                                    # - then, shortly afterwards, but already while we read the file, the
-                                    #   file was changed again, but st2.mtime is the same due to mtime granularity.
-                                    # when comparing file mtime to local clock, widen interval by TIME_DIFFERS1_NS.
-                                    changed_while_backup = True
+                        with backup_io("fstat2"):
+                            st2 = os.fstat(fd)
+                        if self.files_changed == "disabled" or is_special_file:
+                            # special files:
+                            # - fifos change naturally, because they are fed from the other side. no problem.
+                            # - blk/chr devices don't change ctime anyway.
+                            pass
+                        elif self.files_changed == "ctime":
+                            if st.st_ctime_ns != st2.st_ctime_ns:
+                                # ctime was changed, this is either a metadata or a data change.
+                                changed_while_backup = True
+                            elif start_reading - TIME_DIFFERS1_NS < st2.st_ctime_ns < end_reading + TIME_DIFFERS1_NS:
+                                # this is to treat a very special race condition, see #3536.
+                                # - file was changed right before st.ctime was determined.
+                                # - then, shortly afterwards, but already while we read the file, the
+                                #   file was changed again, but st2.ctime is the same due to ctime granularity.
+                                # when comparing file ctime to local clock, widen interval by TIME_DIFFERS1_NS.
+                                changed_while_backup = True
+                        elif self.files_changed == "mtime":
+                            if st.st_mtime_ns != st2.st_mtime_ns:
+                                # mtime was changed, this is either a data change.
+                                changed_while_backup = True
+                            elif start_reading - TIME_DIFFERS1_NS < st2.st_mtime_ns < end_reading + TIME_DIFFERS1_NS:
+                                # this is to treat a very special race condition, see #3536.
+                                # - file was changed right before st.mtime was determined.
+                                # - then, shortly afterwards, but already while we read the file, the
+                                #   file was changed again, but st2.mtime is the same due to mtime granularity.
+                                # when comparing file mtime to local clock, widen interval by TIME_DIFFERS1_NS.
+                                changed_while_backup = True
                         if changed_while_backup:
                             # regular file changed while we backed it up, might be inconsistent/corrupt!
                             if last_try:

src/borg/archiver/create_cmd.py

@@ -252,6 +252,13 @@ def create_inner(archive, cache, fso):
                     nobirthtime=args.nobirthtime,
                 )
                 cp = ChunksProcessor(cache=cache, key=key, add_item=archive.add_item, rechunkify=False)
+                if is_win32 and args.files_changed == "ctime":
+                    self.print_warning(
+                        "--files-changed=ctime is not supported on Windows "
+                        "(ctime is file creation time, not change time). Using mtime instead.",
+                        wc=None,
+                    )
+                    args.files_changed = "mtime"
                 fso = FilesystemObjectProcessors(
                     metadata_collector=metadata_collector,
                     cache=cache,
@@ -621,8 +628,9 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
           well-meant, but in both cases mtime-based cache modes can be problematic.
 
         The ``--files-changed`` option controls how Borg detects if a file has changed during backup:
-         - ctime (default): Use ctime to detect changes. This is the safest option.
-         - mtime: Use mtime to detect changes.
+         - ctime (default on POSIX): Use ctime to detect changes. This is the safest option.
+           Not supported on Windows (ctime is file creation time there).
+         - mtime (default on Windows): Use mtime to detect changes.
          - disabled: Disable the "file has changed while we backed it up" detection completely.
            This is not recommended unless you know what you're doing, as it could lead to
            inconsistent backups if files change during the backup process.
@@ -910,8 +918,9 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
             dest="files_changed",
             action=Highlander,
             choices=["ctime", "mtime", "disabled"],
-            default="ctime",
-            help="specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime",
+            default="mtime" if is_win32 else "ctime",
+            help="specify how to detect if a file has changed during backup (ctime, mtime, disabled). "
+            "default: ctime (on Windows: mtime, because ctime is file creation time there).",
         )
         fs_group.add_argument(
             "--read-special",