feat: add restic (#1044)

* feat: add restic

allows incremental snapshot backups
remove custom push-backup script

* ci: remove .git dir to skip fetch_details_from_tag

fixes https://github.com/frappe/frappe_docker/actions/runs/3938883301/jobs/6738091655

tests/_check_backup_files.py

@@ -1,69 +0,0 @@
-import os
-import re
-from typing import TYPE_CHECKING
-
-import boto3
-
-if TYPE_CHECKING:
-    from mypy_boto3_s3.service_resource import BucketObjectsCollection, _Bucket
-
-
-def get_bucket() -> "_Bucket":
-    return boto3.resource(
-        service_name="s3",
-        endpoint_url="http://minio:9000",
-        region_name="us-east-1",
-        aws_access_key_id=os.getenv("S3_ACCESS_KEY"),
-        aws_secret_access_key=os.getenv("S3_SECRET_KEY"),
-    ).Bucket("frappe")
-
-
-def get_key_builder():
-    site_name = os.getenv("SITE_NAME")
-    assert site_name
-
-    def builder(key: str, suffix: str) -> bool:
-        return bool(re.match(rf"{site_name}.*{suffix}$", key))
-
-    return builder
-
-
-def check_keys(objects: "BucketObjectsCollection"):
-    check_key = get_key_builder()
-
-    db = False
-    config = False
-    private_files = False
-    public_files = False
-
-    for obj in objects:
-        if check_key(obj.key, "database.sql.gz"):
-            db = True
-        elif check_key(obj.key, "site_config_backup.json"):
-            config = True
-        elif check_key(obj.key, "private-files.tar"):
-            private_files = True
-        elif check_key(obj.key, "files.tar"):
-            public_files = True
-
-    exc = lambda type_: Exception(f"Didn't push {type_} backup")
-    if not db:
-        raise exc("database")
-    if not config:
-        raise exc("site config")
-    if not private_files:
-        raise exc("private files")
-    if not public_files:
-        raise exc("public files")
-
-    print("All files were pushed to S3!")
-
-
-def main() -> int:
-    bucket = get_bucket()
-    check_keys(bucket.objects.all())
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

tests/test_frappe_docker.py

@@ -99,40 +99,21 @@ def test_frappe_connections_in_backends(
 
 
 def test_push_backup(
-    python_path: str,
     frappe_site: str,
     s3_service: S3ServiceResult,
     compose: Compose,
 ):
+    restic_password = "secret"
     compose.bench("--site", frappe_site, "backup", "--with-files")
-    compose.exec(
-        "backend",
-        "push-backup",
-        "--site",
-        frappe_site,
-        "--bucket",
-        "frappe",
-        "--region-name",
-        "us-east-1",
-        "--endpoint-url",
-        "http://minio:9000",
-        "--aws-access-key-id",
-        s3_service.access_key,
-        "--aws-secret-access-key",
-        s3_service.secret_key,
-    )
-    compose("cp", "tests/_check_backup_files.py", "backend:/tmp")
-    compose.exec(
-        "-e",
-        f"S3_ACCESS_KEY={s3_service.access_key}",
-        "-e",
-        f"S3_SECRET_KEY={s3_service.secret_key}",
-        "-e",
-        f"SITE_NAME={frappe_site}",
-        "backend",
-        python_path,
-        "/tmp/_check_backup_files.py",
-    )
+    restic_args = [
+        "--env=RESTIC_REPOSITORY=s3:http://minio:9000/frappe",
+        f"--env=AWS_ACCESS_KEY_ID={s3_service.access_key}",
+        f"--env=AWS_SECRET_ACCESS_KEY={s3_service.secret_key}",
+        f"--env=RESTIC_PASSWORD={restic_password}",
+    ]
+    compose.exec(*restic_args, "backend", "restic", "init")
+    compose.exec(*restic_args, "backend", "restic", "backup", "sites")
+    compose.exec(*restic_args, "backend", "restic", "snapshots")
 
 
 def test_https(frappe_site: str, compose: Compose):