Merge remote-tracking branch 'origin/develop' into integrate_last_sign_in

Author: harminius

server/.test.env

@@ -24,3 +24,4 @@ SECURITY_BEARER_SALT='bearer'
 SECURITY_EMAIL_SALT='email'
 SECURITY_PASSWORD_SALT='password'
 DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs
+GEVENT_WORKER=0

server/mergin/sync/config.py

@@ -75,3 +75,6 @@ class Configuration(object):
     UPLOAD_CHUNKS_EXPIRATION = config(
         "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int
     )
+    EXCLUDED_CLONE_FILENAMES = config(
+        "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv()
+    )

server/mergin/sync/permissions.py

@@ -209,7 +209,21 @@ def require_project(ws, project_name, permission) -> Project:
     return project
 
 
-def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled=False):
+def require_project_by_uuid(
+    uuid: str, permission: ProjectPermissions, scheduled=False, expose=True
+) -> Project:
+    """
+    Retrieves a project by UUID after validating existence, workspace status, and permissions.
+
+    Args:
+        uuid (str): The unique identifier of the project.
+        permission (ProjectPermissions): The permission level required to access the project.
+        scheduled (bool, optional): If ``True``, bypasses the check for projects marked for deletion.
+        expose (bool, optional): Controls security disclosure behavior on permission failure.
+            - If `True`: Returns 403 Forbidden (reveals project exists but access is denied).
+            - If `False`: Returns 404 Not Found (hides project existence for security).
+            Standard is that reading results in 404, while writing results in 403
+    """
     if not is_valid_uuid(uuid):
         abort(404)
 
@@ -219,13 +233,18 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled
     if not scheduled:
         project = project.filter(Project.removed_at.is_(None))
     project = project.first_or_404()
+    if not expose and current_user.is_anonymous and not project.public:
+        # we don't want to tell anonymous user if a private project exists
+        abort(404)
+
     workspace = project.workspace
     if not workspace:
         abort(404)
     if not is_active_workspace(workspace):
         abort(404, "Workspace doesn't exist")
     if not permission.check(project, current_user):
         abort(403, "You do not have permissions for this project")
+
     return project
 
 

server/mergin/sync/public_api_controller.py

@@ -1136,9 +1136,12 @@ def clone_project(namespace, project_name):  # noqa: E501
     )
     p.updated = datetime.utcnow()
     db.session.add(p)
+    files_to_exclude = current_app.config.get("EXCLUDED_CLONE_FILENAMES", [])
 
     try:
-        p.storage.initialize(template_project=cloned_project)
+        p.storage.initialize(
+            template_project=cloned_project, excluded_files=files_to_exclude
+        )
     except InitializationError as e:
         abort(400, f"Failed to clone project: {str(e)}")
 
@@ -1149,6 +1152,8 @@ def clone_project(namespace, project_name):  # noqa: E501
     # transform source files to new uploaded files
     file_changes = []
     for file in cloned_project.files:
+        if os.path.basename(file.path) in files_to_exclude:
+            continue
         file_changes.append(
             ProjectFileChange(
                 file.path,

server/mergin/sync/public_api_v2.yaml

@@ -87,8 +87,7 @@ paths:
           description: Include list of files at specific version
           required: false
           schema:
-            type: string
-            example: v3
+            $ref: "#/components/schemas/VersionName"
       responses:
         "200":
           description: Success
@@ -305,9 +304,7 @@ paths:
                   default: false
                   example: true
                 version:
-                  type: string
-                  pattern: '^$|^v\d+$'
-                  example: v2
+                  $ref: "#/components/schemas/VersionName"
                 changes:
                   type: object
                   required:
@@ -333,7 +330,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: "#/components/schemas/Project"
+                $ref: "#/components/schemas/ProjectDetail"
         "204":
           $ref: "#/components/responses/NoContent"
         "400":
@@ -592,7 +589,7 @@ components:
           description: List of files in the project
           items:
             allOf:
-              - $ref: '#/components/schemas/File'
+              - $ref: "#/components/schemas/File"
               - type: object
                 properties:
                   mtime:
@@ -774,81 +771,7 @@ components:
                 type: string
                 format: date-time
                 example: 2019-02-26T08:47:58.636074Z
-    Project:
-      type: object
-      required:
-        - name
-      properties:
-        id:
-          type: string
-          example: f9ef87ac-1dae-48ab-85cb-062a4784fb83
-          description: Project UUID
-        name:
-          type: string
-          example: mergin
-        namespace:
-          type: string
-          example: mergin
-        creator:
-          nullable: true
-          type: integer
-          example: 1
-          description: Project creator ID
-        created:
-          type: string
-          format: date-time
-          example: 2018-11-30T08:47:58.636074Z
-        updated:
-          type: string
-          nullable: true
-          format: date-time
-          example: 2018-11-30T08:47:58.636074Z
-          description: Last modified
-        version:
-          type: string
-          nullable: true
-          example: v2
-          description: Last project version
-        disk_usage:
-          type: integer
-          example: 25324373
-          description: Project size in bytes
-        permissions:
-          type: object
-          properties:
-            delete:
-              type: boolean
-              example: false
-            update:
-              type: boolean
-              example: false
-            upload:
-              type: boolean
-              example: true
-        tags:
-          type: array
-          nullable: true
-          items:
-            $ref: "#/components/schemas/MerginTag"
-        uploads:
-          type: array
-          nullable: true
-          items:
-            type: string
-            example: 669b838e-a30b-4338-b2b6-3da144742a82
-            description: UUID for ongoing upload
-        access:
-          $ref: "#/components/schemas/Access"
-        files:
-          type: array
-          items:
-            allOf:
-              - $ref: "#/components/schemas/FileInfo"
-        role:
-          nullable: true
-          type: string
-          enum:
-            - reader
-            - editor
-            - writer
-            - owner
+    VersionName:
+      type: string
+      pattern: '^$|^v\d+$'
+      example: v2

server/mergin/sync/public_api_v2_controller.py

@@ -44,7 +44,6 @@
 from .public_api_controller import catch_sync_failure
 from .schemas import (
     ProjectMemberSchema,
-    ProjectVersionSchema,
     UploadChunkSchema,
     ProjectSchema,
 )
@@ -168,7 +167,7 @@ def remove_project_collaborator(id, user_id):
 
 def get_project(id, files_at_version=None):
     """Get project info. Include list of files at specific version if requested."""
-    project = require_project_by_uuid(id, ProjectPermissions.Read)
+    project = require_project_by_uuid(id, ProjectPermissions.Read, expose=False)
     data = ProjectSchemaV2().dump(project)
     if files_at_version:
         pv = ProjectVersion.query.filter_by(
@@ -362,7 +361,12 @@ def create_project_version(id):
     finally:
         # remove artifacts
         upload.clear()
-    return ProjectSchema().dump(project), 201
+
+    result = ProjectSchemaV2().dump(project)
+    result["files"] = ProjectFileSchema(
+        only=("path", "mtime", "size", "checksum"), many=True
+    ).dump(project.files)
+    return result, 201
 
 
 @auth_required

server/mergin/sync/storages/disk.py

@@ -178,7 +178,7 @@ def _project_dir(self):
         )
         return project_dir
 
-    def initialize(self, template_project=None):
+    def initialize(self, template_project=None, excluded_files=None):
         if os.path.exists(self.project_dir):
             raise InitializationError(
                 "Project directory already exists: {}".format(self.project_dir)
@@ -193,8 +193,12 @@ def initialize(self, template_project=None):
             if ws.disk_usage() + template_project.disk_usage > ws.storage:
                 self.delete()
                 raise InitializationError("Disk quota reached")
+            if excluded_files is None:
+                excluded_files = []
 
             for file in template_project.files:
+                if os.path.basename(file.path) in excluded_files:
+                    continue
                 src = os.path.join(template_project.storage.project_dir, file.location)
                 dest = os.path.join(
                     self.project_dir,

server/mergin/tests/test_middleware.py

@@ -6,6 +6,7 @@
 import psycogreen.gevent
 import pytest
 import sqlalchemy
+from unittest.mock import patch
 
 from ..app import create_simple_app, GeventTimeoutMiddleware, db
 from ..config import Configuration
@@ -14,58 +15,74 @@
 @pytest.mark.parametrize("use_middleware", [True, False])
 def test_use_middleware(use_middleware):
     """Test using middleware"""
-    Configuration.GEVENT_WORKER = use_middleware
-    Configuration.GEVENT_REQUEST_TIMEOUT = 1
-    application = create_simple_app()
+    with patch.object(
+        Configuration,
+        "GEVENT_WORKER",
+        use_middleware,
+    ), patch.object(
+        Configuration,
+        "GEVENT_REQUEST_TIMEOUT",
+        1,
+    ):
+        application = create_simple_app()
 
-    def ping():
-        gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1)
-        return "pong"
+        def ping():
+            gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1)
+            return "pong"
 
-    application.add_url_rule("/test", "ping", ping)
-    app_context = application.app_context()
-    app_context.push()
+        application.add_url_rule("/test", "ping", ping)
+        app_context = application.app_context()
+        app_context.push()
 
-    assert isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware
-    # in case of gevent, dummy endpoint it set to time out
-    assert application.test_client().get("/test").status_code == (
-        504 if use_middleware else 200
-    )
+        assert (
+            isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware
+        )
+        # in case of gevent, dummy endpoint it set to time out
+        assert application.test_client().get("/test").status_code == (
+            504 if use_middleware else 200
+        )
 
 
 def test_catch_timeout():
     """Test proper handling of gevent timeout with db.session.rollback"""
     psycogreen.gevent.patch_psycopg()
-    Configuration.GEVENT_WORKER = True
-    Configuration.GEVENT_REQUEST_TIMEOUT = 1
-    application = create_simple_app()
+    with patch.object(
+        Configuration,
+        "GEVENT_WORKER",
+        True,
+    ), patch.object(
+        Configuration,
+        "GEVENT_REQUEST_TIMEOUT",
+        1,
+    ):
+        application = create_simple_app()
 
-    def unhandled():
-        try:
-            db.session.execute("SELECT pg_sleep(1.1);")
-        finally:
-            db.session.execute("SELECT 1;")
-        return ""
+        def unhandled():
+            try:
+                db.session.execute("SELECT pg_sleep(1.1);")
+            finally:
+                db.session.execute("SELECT 1;")
+            return ""
 
-    def timeout():
-        try:
-            db.session.execute("SELECT pg_sleep(1.1);")
-        except gevent.timeout.Timeout:
-            db.session.rollback()
-            raise
-        finally:
-            db.session.execute("SELECT 1;")
-        return ""
+        def timeout():
+            try:
+                db.session.execute("SELECT pg_sleep(1.1);")
+            except gevent.timeout.Timeout:
+                db.session.rollback()
+                raise
+            finally:
+                db.session.execute("SELECT 1;")
+            return ""
 
-    application.add_url_rule("/unhandled", "unhandled", unhandled)
-    application.add_url_rule("/timeout", "timeout", timeout)
-    app_context = application.app_context()
-    app_context.push()
+        application.add_url_rule("/unhandled", "unhandled", unhandled)
+        application.add_url_rule("/timeout", "timeout", timeout)
+        app_context = application.app_context()
+        app_context.push()
 
-    assert application.test_client().get("/timeout").status_code == 504
+        assert application.test_client().get("/timeout").status_code == 504
 
-    # in case of missing rollback sqlalchemy would raise error
-    with pytest.raises(sqlalchemy.exc.PendingRollbackError):
-        application.test_client().get("/unhandled")
+        # in case of missing rollback sqlalchemy would raise error
+        with pytest.raises(sqlalchemy.exc.PendingRollbackError):
+            application.test_client().get("/unhandled")
 
-    db.session.rollback()
+        db.session.rollback()