Fix sync loop for too long path

mergin/client.py

@@ -2,7 +2,6 @@
 import math
 import os
 import json
-import shutil
 import zlib
 import base64
 import urllib.parse
@@ -69,7 +68,9 @@
     int_version,
     is_version_acceptable,
     normalize_role,
+    long_path,
 )
+from . import fs
 from .version import __version__
 
 try:
@@ -207,7 +208,7 @@ def setup_logging(self):
         self.log.setLevel(logging.DEBUG)  # log everything (it would otherwise log just warnings+errors)
         if not self.log.handlers:
             if client_log_file:
-                log_handler = logging.FileHandler(client_log_file)
+                log_handler = logging.FileHandler(long_path(client_log_file))
                 log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
                 self.log.addHandler(log_handler)
             else:
@@ -534,7 +535,7 @@ def create_project_and_push(self, project_name, directory, is_public=False, name
         :param namespace: Deprecated. project_name should be full project name. Optional namespace for a new project. If empty username is used.
         :type namespace: String
         """
-        if os.path.exists(os.path.join(directory, ".mergin")):
+        if fs.exists(os.path.join(directory, ".mergin")):
             raise ClientError("Directory is already assigned to a Mergin Maps project (contains .mergin sub-dir)")
 
         if namespace and "/" not in project_name:
@@ -1276,11 +1277,11 @@ def get_file_diff(self, project_dir, file_path, output_diff, version_from, versi
         # concatenate diffs, if needed
         output_dir = os.path.abspath(os.path.dirname(output_diff))
         if len(diffs) >= 1:
-            os.makedirs(output_dir, exist_ok=True)
+            fs.makedirs(output_dir, exist_ok=True)
             if len(diffs) > 1:
                 mp.geodiff.concat_changes(diffs, output_diff)
             elif len(diffs) == 1:
-                shutil.copy(diffs[0], output_diff)
+                fs.copy(diffs[0], output_diff)
 
     def download_file_diffs(self, project_dir, file_path, versions):
         """Download file diffs for specified versions if they are not present
@@ -1411,7 +1412,7 @@ def reset_local_changes(self, directory: str, files_to_reset: typing.List[str] =
         # remove all added files
         for file in push_changes["added"]:
             if all_files or file["path"] in files_to_reset:
-                os.remove(mp.fpath(file["path"]))
+                fs.remove(mp.fpath(file["path"]))
 
         # update files get override with previous version
         for file in push_changes["updated"]:
@@ -1646,14 +1647,14 @@ def send_logs(
         local_logs_file_size_to_send = int(MAX_LOG_FILE_SIZE_TO_SEND * 0.8)
 
         global_logs = b""
-        if global_log_file and os.path.exists(global_log_file):
-            with open(global_log_file, "rb") as f:
-                if os.path.getsize(global_log_file) > global_logs_file_size_to_send:
+        if global_log_file and fs.exists(global_log_file):
+            with fs.open_file(global_log_file, "rb") as f:
+                if fs.getsize(global_log_file) > global_logs_file_size_to_send:
                     f.seek(-global_logs_file_size_to_send, os.SEEK_END)
                 global_logs = f.read() + b"\n--------------------------------\n\n"
 
-        with open(logfile, "rb") as f:
-            if os.path.getsize(logfile) > local_logs_file_size_to_send:
+        with fs.open_file(logfile, "rb") as f:
+            if fs.getsize(logfile) > local_logs_file_size_to_send:
                 f.seek(-local_logs_file_size_to_send, os.SEEK_END)
             logs = f.read()
 

mergin/client_pull.py

@@ -25,6 +25,7 @@
 from .common import CHUNK_SIZE, ClientError, DeltaChangeType, PullActionType
 from .models import ProjectDelta, ProjectDeltaChange, PullAction
 from .merginproject import MerginProject
+from . import fs
 from .utils import cleanup_tmp_dir, filter_files, is_path_in_scope, save_to_file, is_versioned_file
 from typing import List, Optional, Union
 
@@ -161,7 +162,7 @@ def download_blocking(self, mc, mp):
         if resp.status in [200, 206]:
             mp.log.debug(f"Download finished: {self.diff_id}")
             save_to_file(resp, self.download_file_path)
-            self.size = os.path.getsize(self.download_file_path)
+            self.size = fs.getsize(self.download_file_path)
         else:
             mp.log.error(f"Download failed: {self.diff_id}")
             raise ClientError(f"Failed to download of diff file {self.diff_id} to {self.download_file_path}")
@@ -183,19 +184,19 @@ def __init__(self, dest_file, downloaded_items: typing.List[DownloadQueueItem],
     def from_chunks(self):
         """Merges downloaded chunks into a single file at dest_file path"""
         file_dir = os.path.abspath(os.path.dirname(self.dest_file))
-        os.makedirs(file_dir, exist_ok=True)
+        fs.makedirs(file_dir, exist_ok=True)
 
-        with open(self.dest_file, "wb") as final:
+        with fs.open_file(self.dest_file, "wb") as final:
             for item in self.downloaded_items:
-                with open(item.download_file_path, "rb") as chunk:
+                with fs.open_file(item.download_file_path, "rb") as chunk:
                     shutil.copyfileobj(chunk, final)
-                os.remove(item.download_file_path)
+                fs.remove(item.download_file_path)
 
         if not self.size_check:
             return
         expected_size = sum(item.size for item in self.downloaded_items)
-        if os.path.getsize(self.dest_file) != expected_size:
-            os.remove(self.dest_file)
+        if fs.getsize(self.dest_file) != expected_size:
+            fs.remove(self.dest_file)
             raise ClientError("Download of file {} failed. Please try it again.".format(self.dest_file))
 
 
@@ -252,7 +253,7 @@ def _cleanup_failed_download(mergin_project: MerginProject = None):
     log_file = os.path.join(mergin_project.dir, ".mergin", "client-log.txt")
     dest_path = None
 
-    if os.path.exists(log_file):
+    if fs.exists(log_file):
         tmp_file = tempfile.NamedTemporaryFile(prefix="mergin-", suffix=".txt", delete=False)
         tmp_file.close()
         dest_path = tmp_file.name
@@ -275,9 +276,9 @@ def download_project_async(mc, project_path, directory, project_version=None, in
         raise ClientError("Cannot use both include and exclude filters at the same time")
     if "/" not in project_path:
         raise ClientError("Project name needs to be fully qualified, e.g. <username>/<projectname>")
-    if os.path.exists(directory):
+    if fs.exists(directory):
         raise ClientError("Project directory already exists")
-    os.makedirs(directory)
+    fs.makedirs(directory)
     mp = MerginProject(directory)
 
     mp.log.info("--- version: " + mc.user_agent_info())
@@ -439,7 +440,7 @@ def apply(self, directory, mp: Union[MerginProject, "DownloadScratchContext"]):
         else:
             file_dir = os.path.dirname(os.path.abspath(self.destination_file))
             dest_file_path = self.destination_file
-        os.makedirs(file_dir, exist_ok=True)
+        fs.makedirs(file_dir, exist_ok=True)
 
         # ignore check if we download not-latest version of gpkg file (possibly reconstructed on server on demand)
         check_size = self.latest_version or not is_versioned_file(self.file_path)
@@ -589,7 +590,7 @@ def pull_project_async(mc, directory) -> Optional[PullJob]:
             pull_action_type == PullActionType.COPY_CONFLICT and change.type == DeltaChangeType.UPDATE_DIFF
         ):
             basefile = mp.fpath_meta(change.path)
-            if not os.path.exists(basefile):
+            if not fs.exists(basefile):
                 # The basefile does not exist for some reason. This should not happen normally (maybe user removed the file
                 # or we removed it within previous pull because we failed to apply patch the older version for some reason).
                 # But it's not a problem - we will download the newest version and we're sorted.
@@ -756,7 +757,7 @@ def pull_project_finalize(job: PullJob):
         basefile = job.mp.fpath_meta(file_path)
         server_file = job.mp.fpath(file_path, job.tmp_dir.name)
 
-        shutil.copy(basefile, server_file)
+        fs.copy(basefile, server_file)
         diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs]
         patch_error = job.mp.apply_diffs(server_file, diffs)
         if patch_error:
@@ -769,7 +770,7 @@ def pull_project_finalize(job: PullJob):
             job.mp.log.error("Diffs we were applying: " + str(diffs))
             job.mp.log.error("Removing basefile because it would be corrupted anyway...")
             job.mp.log.info("--- pull aborted")
-            os.remove(basefile)
+            fs.remove(basefile)
             raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile))
     conflicts = []
     job.mp.log.info(f"--- applying pull actions {job.pull_actions}")
@@ -884,7 +885,7 @@ def download_diffs_async(mc, project_directory, file_path, versions):
             diff_only=True,
         )
         dest_file_path = mp.fpath_cache(diff["path"], version=file["version"])
-        if os.path.exists(dest_file_path):
+        if fs.exists(dest_file_path):
             continue
         download_files.append(DownloadFile(dest_file_path, items))
         download_list.extend(items)
@@ -1057,4 +1058,6 @@ def download_files_finalize(job: DownloadJob):
     for task in job.update_tasks:
         task.apply(job.tmp_dir, job.mp)
 
-    cleanup_tmp_dir(job.mp, job.tmp_dir)
+    # Remove temporary download directory
+    if job.tmp_dir is not None and fs.exists(job.tmp_dir.name):
+        cleanup_tmp_dir(job.mp, job.tmp_dir)

mergin/client_push.py

@@ -18,7 +18,6 @@
 import pprint
 import tempfile
 import concurrent.futures
-import os
 import time
 from typing import List, Tuple, Optional, ByteString
 
@@ -35,6 +34,7 @@
 from .merginproject import MerginProject, pygeodiff
 from .editor import filter_changes
 from .utils import get_data_checksum, cleanup_tmp_dir, filter_files
+from . import fs
 
 POST_JSON_HEADERS = {"Content-Type": "application/json"}
 
@@ -114,7 +114,7 @@ def upload_chunk_v2_api(self, data: ByteString, checksum: str):
         self.mc.upload_chunks_cache.add(checksum, self.server_chunk_id)
 
     def upload_blocking(self):
-        with open(self.file_path, "rb") as file_handle:
+        with fs.open_file(self.file_path, "rb") as file_handle:
             file_handle.seek(self.chunk_index * UPLOAD_CHUNK_SIZE)
             data = file_handle.read(UPLOAD_CHUNK_SIZE)
             checksum_str = get_data_checksum(data)
@@ -511,8 +511,8 @@ def remove_diff_files(job: UploadJob) -> None:
         diff = change.get_diff()
         if diff:
             diff_file = job.mp.fpath_meta(diff.path)
-            if os.path.exists(diff_file):
-                os.remove(diff_file)
+            if fs.exists(diff_file):
+                fs.remove(diff_file)
 
 
 def get_push_changes_batch(mc, directory: str) -> Tuple[LocalProjectChanges, int]:

mergin/fs.py

@@ -0,0 +1,59 @@
+"""
+Thin wrappers around the standard-library filesystem calls used by the sync code.
+
+Every wrapper applies utils.long_path() to its path argument(s) so that Windows paths
+longer than MAX_PATH are handled transparently.
+
+Use these instead of calling os.* / shutil.* / open() / sqlite3.connect() directly on
+project file paths.
+"""
+
+import os
+import shutil
+import sqlite3
+
+from .utils import long_path
+
+
+def remove(path):
+    os.remove(long_path(path))
+
+
+def exists(path) -> bool:
+    return os.path.exists(long_path(path))
+
+
+def getsize(path) -> int:
+    return os.path.getsize(long_path(path))
+
+
+def getmtime(path) -> float:
+    return os.path.getmtime(long_path(path))
+
+
+def copy(src, dst):
+    return shutil.copy(long_path(src), long_path(dst))
+
+
+def walk(path, **kwargs):
+    return os.walk(long_path(path), **kwargs)
+
+
+def makedirs(path, exist_ok=False):
+    os.makedirs(long_path(path), exist_ok=exist_ok)
+
+
+def mkdir(path):
+    os.mkdir(long_path(path))
+
+
+def rmtree(path):
+    shutil.rmtree(long_path(path))
+
+
+def connect(path):
+    return sqlite3.connect(long_path(path))
+
+
+def open_file(path, *args, **kwargs):
+    return open(long_path(path), *args, **kwargs)