refactor: build only one frappe/erpnext image (#1032)
* ci: skip frappe builds * refactor: build only one frappe/erpnext image * fix: lint nginx entrypoint script * docs: update and organize docs * docs: fix lint errors * fix(custom): pass base64 encoded apps json * ci: update dependabot * docs: update contributing * docs: remove info about multi image setup * fix: initiate empty common_site_config.json default config has host keys set to localhost causes connection errors * docs: add details for pwd volumes * fix: symlink assets instead of copy * fix: nginx private files * ci: skip docker compose v2 install for ubuntu-latest * fix: organize layers * feat: allow remove git remote for custom image * docs: allow remove git remote for custom image * fix: remove duplicate --apps_path
This commit is contained in:
Revant Nandgaonkar
committed by
GitHub
GitHub
parent
f8e43a3114
commit
e6088af885
52
resources/nginx-entrypoint.sh
Executable file
52
resources/nginx-entrypoint.sh
Executable file
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Set variables that do not exist
|
||||
if [[ -z "$BACKEND" ]]; then
|
||||
echo "BACKEND defaulting to 0.0.0.0:8000"
|
||||
export BACKEND=0.0.0.0:8000
|
||||
fi
|
||||
if [[ -z "$SOCKETIO" ]]; then
|
||||
echo "SOCKETIO defaulting to 0.0.0.0:9000"
|
||||
export SOCKETIO=0.0.0.0:9000
|
||||
fi
|
||||
if [[ -z "$UPSTREAM_REAL_IP_ADDRESS" ]]; then
|
||||
echo "UPSTREAM_REAL_IP_ADDRESS defaulting to 127.0.0.1"
|
||||
export UPSTREAM_REAL_IP_ADDRESS=127.0.0.1
|
||||
fi
|
||||
if [[ -z "$UPSTREAM_REAL_IP_HEADER" ]]; then
|
||||
echo "UPSTREAM_REAL_IP_HEADER defaulting to X-Forwarded-For"
|
||||
export UPSTREAM_REAL_IP_HEADER=X-Forwarded-For
|
||||
fi
|
||||
if [[ -z "$UPSTREAM_REAL_IP_RECURSIVE" ]]; then
|
||||
echo "UPSTREAM_REAL_IP_RECURSIVE defaulting to off"
|
||||
export UPSTREAM_REAL_IP_RECURSIVE=off
|
||||
fi
|
||||
if [[ -z "$FRAPPE_SITE_NAME_HEADER" ]]; then
|
||||
# shellcheck disable=SC2016
|
||||
echo 'FRAPPE_SITE_NAME_HEADER defaulting to $host'
|
||||
# shellcheck disable=SC2016
|
||||
export FRAPPE_SITE_NAME_HEADER='$host'
|
||||
fi
|
||||
|
||||
if [[ -z "$PROXY_READ_TIMEOUT" ]]; then
|
||||
echo "PROXY_READ_TIMEOUT defaulting to 120"
|
||||
export PROXY_READ_TIMEOUT=120
|
||||
fi
|
||||
|
||||
if [[ -z "$CLIENT_MAX_BODY_SIZE" ]]; then
|
||||
echo "CLIENT_MAX_BODY_SIZE defaulting to 50m"
|
||||
export CLIENT_MAX_BODY_SIZE=50m
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2016
|
||||
envsubst '${BACKEND}
|
||||
${SOCKETIO}
|
||||
${UPSTREAM_REAL_IP_ADDRESS}
|
||||
${UPSTREAM_REAL_IP_HEADER}
|
||||
${UPSTREAM_REAL_IP_RECURSIVE}
|
||||
${FRAPPE_SITE_NAME_HEADER}
|
||||
${PROXY_READ_TIMEOUT}
|
||||
${CLIENT_MAX_BODY_SIZE}' \
|
||||
</templates/nginx/frappe.conf.template >/etc/nginx/conf.d/frappe.conf
|
||||
|
||||
nginx -g 'daemon off;'
|
||||
114
resources/nginx-template.conf
Normal file
114
resources/nginx-template.conf
Normal file
@@ -0,0 +1,114 @@
|
||||
upstream backend-server {
|
||||
server ${BACKEND} fail_timeout=0;
|
||||
}
|
||||
|
||||
upstream socketio-server {
|
||||
server ${SOCKETIO} fail_timeout=0;
|
||||
}
|
||||
|
||||
# Parse the X-Forwarded-Proto header - if set - defaulting to $scheme.
|
||||
map $http_x_forwarded_proto $proxy_x_forwarded_proto {
|
||||
default $scheme;
|
||||
https https;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name ${FRAPPE_SITE_NAME_HEADER};
|
||||
root /home/frappe/frappe-bench/sites;
|
||||
|
||||
proxy_buffer_size 128k;
|
||||
proxy_buffers 4 256k;
|
||||
proxy_busy_buffers_size 256k;
|
||||
|
||||
add_header X-Frame-Options "SAMEORIGIN";
|
||||
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload";
|
||||
add_header X-Content-Type-Options nosniff;
|
||||
add_header X-XSS-Protection "1; mode=block";
|
||||
add_header Referrer-Policy "same-origin, strict-origin-when-cross-origin";
|
||||
|
||||
set_real_ip_from ${UPSTREAM_REAL_IP_ADDRESS};
|
||||
real_ip_header ${UPSTREAM_REAL_IP_HEADER};
|
||||
real_ip_recursive ${UPSTREAM_REAL_IP_RECURSIVE};
|
||||
|
||||
location /assets {
|
||||
try_files $uri =404;
|
||||
}
|
||||
|
||||
location ~ ^/protected/(.*) {
|
||||
internal;
|
||||
try_files /${FRAPPE_SITE_NAME_HEADER}/$1 =404;
|
||||
}
|
||||
|
||||
location /socket.io {
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER};
|
||||
proxy_set_header Origin $scheme://${FRAPPE_SITE_NAME_HEADER};
|
||||
proxy_set_header Host $host;
|
||||
|
||||
proxy_pass http://socketio-server;
|
||||
}
|
||||
|
||||
location / {
|
||||
rewrite ^(.+)/$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent;
|
||||
rewrite ^(.+)/index\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent;
|
||||
rewrite ^(.+)\.html$ $proxy_x_forwarded_proto://${FRAPPE_SITE_NAME_HEADER}$1 permanent;
|
||||
|
||||
location ~ ^/files/.*.(htm|html|svg|xml) {
|
||||
add_header Content-disposition "attachment";
|
||||
try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver;
|
||||
}
|
||||
|
||||
try_files /${FRAPPE_SITE_NAME_HEADER}/public/$uri @webserver;
|
||||
}
|
||||
|
||||
location @webserver {
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $proxy_x_forwarded_proto;
|
||||
proxy_set_header X-Frappe-Site-Name ${FRAPPE_SITE_NAME_HEADER};
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Use-X-Accel-Redirect True;
|
||||
proxy_read_timeout ${PROXY_READ_TIMEOUT};
|
||||
proxy_redirect off;
|
||||
|
||||
proxy_pass http://backend-server;
|
||||
}
|
||||
|
||||
# optimizations
|
||||
sendfile on;
|
||||
keepalive_timeout 15;
|
||||
client_max_body_size ${CLIENT_MAX_BODY_SIZE};
|
||||
client_body_buffer_size 16K;
|
||||
client_header_buffer_size 1k;
|
||||
|
||||
# enable gzip compression
|
||||
# based on https://mattstauffer.co/blog/enabling-gzip-on-nginx-servers-including-laravel-forge
|
||||
gzip on;
|
||||
gzip_http_version 1.1;
|
||||
gzip_comp_level 5;
|
||||
gzip_min_length 256;
|
||||
gzip_proxied any;
|
||||
gzip_vary on;
|
||||
gzip_types
|
||||
application/atom+xml
|
||||
application/javascript
|
||||
application/json
|
||||
application/rss+xml
|
||||
application/vnd.ms-fontobject
|
||||
application/x-font-ttf
|
||||
application/font-woff
|
||||
application/x-web-app-manifest+json
|
||||
application/xhtml+xml
|
||||
application/xml
|
||||
font/opentype
|
||||
image/svg+xml
|
||||
image/x-icon
|
||||
text/css
|
||||
text/plain
|
||||
text/x-component;
|
||||
# text/html is always compressed by HttpGzipModule
|
||||
}
|
||||
118
resources/push_backup.py
Executable file
118
resources/push_backup.py
Executable file
@@ -0,0 +1,118 @@
|
||||
#!/home/frappe/frappe-bench/env/bin/python
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, List, cast
|
||||
|
||||
import boto3
|
||||
import frappe
|
||||
from frappe.utils.backups import BackupGenerator
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from mypy_boto3_s3.service_resource import _Bucket
|
||||
|
||||
|
||||
class Arguments(argparse.Namespace):
|
||||
site: str
|
||||
bucket: str
|
||||
region_name: str
|
||||
endpoint_url: str
|
||||
aws_access_key_id: str
|
||||
aws_secret_access_key: str
|
||||
bucket_directory: str
|
||||
|
||||
|
||||
def _get_files_from_previous_backup(site_name: str) -> list[Path]:
|
||||
frappe.connect(site_name)
|
||||
|
||||
conf = cast(Any, frappe.conf)
|
||||
backup_generator = BackupGenerator(
|
||||
db_name=conf.db_name,
|
||||
user=conf.db_name,
|
||||
password=conf.db_password,
|
||||
db_host=frappe.db.host,
|
||||
db_port=frappe.db.port,
|
||||
db_type=conf.db_type,
|
||||
)
|
||||
recent_backup_files = backup_generator.get_recent_backup(24)
|
||||
|
||||
frappe.destroy()
|
||||
return [Path(f) for f in recent_backup_files if f]
|
||||
|
||||
|
||||
def get_files_from_previous_backup(site_name: str) -> list[Path]:
|
||||
files = _get_files_from_previous_backup(site_name)
|
||||
if not files:
|
||||
print("No backup found that was taken <24 hours ago.")
|
||||
return files
|
||||
|
||||
|
||||
def get_bucket(args: Arguments) -> _Bucket:
|
||||
return boto3.resource(
|
||||
service_name="s3",
|
||||
endpoint_url=args.endpoint_url,
|
||||
region_name=args.region_name,
|
||||
aws_access_key_id=args.aws_access_key_id,
|
||||
aws_secret_access_key=args.aws_secret_access_key,
|
||||
).Bucket(args.bucket)
|
||||
|
||||
|
||||
def upload_file(
|
||||
path: Path, site_name: str, bucket: _Bucket, bucket_directory: str = None
|
||||
) -> None:
|
||||
filename = str(path.absolute())
|
||||
key = str(Path(site_name) / path.name)
|
||||
if bucket_directory:
|
||||
key = bucket_directory + "/" + key
|
||||
print(f"Uploading {key}")
|
||||
bucket.upload_file(Filename=filename, Key=key)
|
||||
os.remove(path)
|
||||
|
||||
|
||||
def push_backup(args: Arguments) -> None:
|
||||
"""Get latest backup files using Frappe utils, push them to S3 and remove local copy"""
|
||||
|
||||
files = get_files_from_previous_backup(args.site)
|
||||
bucket = get_bucket(args)
|
||||
|
||||
for path in files:
|
||||
upload_file(
|
||||
path=path,
|
||||
site_name=args.site,
|
||||
bucket=bucket,
|
||||
bucket_directory=args.bucket_directory,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
|
||||
|
||||
def parse_args(args: list[str]) -> Arguments:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--site", required=True)
|
||||
parser.add_argument("--bucket", required=True)
|
||||
parser.add_argument("--region-name", required=True)
|
||||
parser.add_argument("--endpoint-url", required=True)
|
||||
# Looking for default AWS credentials variables
|
||||
parser.add_argument(
|
||||
"--aws-access-key-id", required=True, default=os.getenv("AWS_ACCESS_KEY_ID")
|
||||
)
|
||||
parser.add_argument(
|
||||
"--aws-secret-access-key",
|
||||
required=True,
|
||||
default=os.getenv("AWS_SECRET_ACCESS_KEY"),
|
||||
)
|
||||
parser.add_argument("--bucket-directory")
|
||||
return parser.parse_args(args, namespace=Arguments())
|
||||
|
||||
|
||||
def main(args: list[str]) -> int:
|
||||
os.chdir("sites")
|
||||
push_backup(parse_args(args))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
Reference in New Issue
Block a user