Skip to content

Commit

Permalink
[Fixes #11977] B/R should only deal with data (#12007)
Browse files Browse the repository at this point in the history
  • Loading branch information
@etj
etj committed Mar 6, 2024
1 parent 3df7b47 commit 3c66a54
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 266 deletions.
125 changes: 4 additions & 121 deletions geonode/br/management/commands/backup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,14 +156,17 @@ def execute_backup(self, **options):
# Dump Fixtures
logger.info("*** Dumping GeoNode fixtures...")

fixtures_target = os.path.join(target_folder, "fixtures")
os.makedirs(fixtures_target, exist_ok=True)

for app_name, dump_name in zip(config.app_names, config.dump_names):
# prevent dumping BackupRestore application
if app_name == "br":
continue

logger.info(f" - Dumping '{app_name}' into '{dump_name}.json'")
# Point stdout at a file for dumping data to.
with open(os.path.join(target_folder, f"{dump_name}.json"), "w") as output:
with open(os.path.join(fixtures_target, f"{dump_name}.json"), "w") as output:
call_command("dumpdata", app_name, format="json", indent=2, stdout=output)

# Store Media Root
Expand All @@ -181,126 +184,6 @@ def execute_backup(self, **options):
)
logger.info(f"Saved media files from '{media_root}'")

# Store Static Root
logger.info("*** Dumping GeoNode static folder...")

static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, utils.STATIC_ROOT)
if not os.path.exists(static_folder):
os.makedirs(static_folder, exist_ok=True)

copy_tree(
static_root,
static_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]),
)
logger.info(f"Saved static root from '{static_root}'.")

# Store Static Folders
logger.info("*** Dumping GeoNode static files...")

static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, utils.STATICFILES_DIRS)
if not os.path.exists(static_files_folders):
os.makedirs(static_files_folders, exist_ok=True)

for static_files_folder in static_folders:
# skip dumping of static files of apps not located under PROJECT_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not static_files_folder.startswith(
settings.PROJECT_ROOT
):
logger.info(
f"Skipping static directory: {static_files_folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

static_folder = os.path.join(
static_files_folders, os.path.basename(os.path.normpath(static_files_folder))
)
if not os.path.exists(static_folder):
os.makedirs(static_folder, exist_ok=True)

copy_tree(
static_files_folder,
static_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]),
)
logger.info(f"Saved static files from '{static_files_folder}'.")

# Store Template Folders
logger.info("*** Dumping GeoNode template folders...")

template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except Exception:
try:
template_folders = settings.TEMPLATES[0]["DIRS"]
except Exception:
pass
template_files_folders = os.path.join(target_folder, utils.TEMPLATE_DIRS)
if not os.path.exists(template_files_folders):
os.makedirs(template_files_folders, exist_ok=True)

for template_files_folder in template_folders:
# skip dumping of template files of apps not located under PROJECT_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not template_files_folder.startswith(
settings.PROJECT_ROOT
):
logger.info(
f"Skipping template directory: {template_files_folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

template_folder = os.path.join(
template_files_folders, os.path.basename(os.path.normpath(template_files_folder))
)
if not os.path.exists(template_folder):
os.makedirs(template_folder, exist_ok=True)

copy_tree(
template_files_folder,
template_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]),
)
logger.info(f"Saved template files from '{template_files_folder}'.")

# Store Locale Folders
logger.info("*** Dumping GeoNode locale folders...")
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, utils.LOCALE_PATHS)
if not os.path.exists(locale_files_folders):
os.makedirs(locale_files_folders, exist_ok=True)

for locale_files_folder in locale_folders:
# skip dumping of locale files of apps not located under PROJECT_ROOT path
# (check to prevent saving files from site-packages in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not locale_files_folder.startswith(
settings.PROJECT_ROOT
):
logger.info(
f"Skipping locale directory: {locale_files_folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

locale_folder = os.path.join(
locale_files_folders, os.path.basename(os.path.normpath(locale_files_folder))
)
if not os.path.exists(locale_folder):
os.makedirs(locale_folder, exist_ok=True)

copy_tree(
locale_files_folder,
locale_folder,
ignore=utils.ignore_time(config.gs_data_dt_filter[0], config.gs_data_dt_filter[1]),
)
logger.info(f"Saved Locale Files from '{locale_files_folder}'.")

# Create Final ZIP Archive
logger.info("*** Creating final ZIP archive...")

Expand Down
161 changes: 16 additions & 145 deletions geonode/br/management/commands/restore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,11 @@ def add_arguments(self, parser):
# Named (optional) arguments
utils.option(parser)

utils.geoserver_option_list(parser)
parser.add_argument(
"--geoserver-data-dir",
dest="gs_data_dir",
default=None,
help="Geoserver data directory")

parser.add_argument(
"-i",
Expand Down Expand Up @@ -107,7 +111,7 @@ def add_arguments(self, parser):
"--recovery-file",
dest="recovery_file",
default=None,
help="Backup archive containing GeoNode data to restore.",
help="Archive that shall be used to restore the original content of GeoNode should the restore fail.",
)

parser.add_argument(
Expand Down Expand Up @@ -165,7 +169,7 @@ def handle(self, **options):
skip_read_only = options.get("skip_read_only")
config = Configuration.load()

# activate read only mode and store it's original config value
# activate read only mode and store its original config value
if not skip_read_only:
original_read_only_value = config.read_only
config.read_only = True
Expand Down Expand Up @@ -261,42 +265,13 @@ def execute_restore(self, **options):
# Write Checks
media_root = settings.MEDIA_ROOT
media_folder = os.path.join(target_folder, utils.MEDIA_ROOT)
static_root = settings.STATIC_ROOT
static_folder = os.path.join(target_folder, utils.STATIC_ROOT)
static_folders = settings.STATICFILES_DIRS
static_files_folders = os.path.join(target_folder, utils.STATICFILES_DIRS)
template_folders = []
try:
template_folders = settings.TEMPLATE_DIRS
except Exception:
try:
template_folders = settings.TEMPLATES[0]["DIRS"]
except Exception:
pass
template_files_folders = os.path.join(target_folder, utils.TEMPLATE_DIRS)
locale_folders = settings.LOCALE_PATHS
locale_files_folders = os.path.join(target_folder, utils.LOCALE_PATHS)

try:
logger.info("*** Performing some checks...")
logger.info(f"[Sanity Check] Full Write Access to restore folder: '{restore_folder}' ...")
chmod_tree(restore_folder)
logger.info(f"[Sanity Check] Full Write Access to media root: '{media_root}' ...")
chmod_tree(media_root)
logger.info(f"[Sanity Check] Full Write Access to static root: '{static_root}' ...")
chmod_tree(static_root)
for folder in static_folders:
if getattr(settings, "PROJECT_ROOT", None) and folder.startswith(settings.PROJECT_ROOT):
logger.info(f"[Sanity Check] Full Write Access to static file folder: '{folder}' ...")
chmod_tree(folder)
for folder in template_folders:
if getattr(settings, "PROJECT_ROOT", None) and folder.startswith(settings.PROJECT_ROOT):
logger.info(f"[Sanity Check] Full Write Access to template folder: '{folder}' ...")
chmod_tree(folder)
for folder in locale_folders:
if getattr(settings, "PROJECT_ROOT", None) and folder.startswith(settings.PROJECT_ROOT):
logger.info(f"[Sanity Check] Full Write Access to locale files folder: '{folder}' ...")
chmod_tree(folder)
except Exception as e:
if notify:
restore_notification.apply_async(
Expand Down Expand Up @@ -361,14 +336,6 @@ def execute_restore(self, **options):
logger.info("*** Align the database schema")
# call_command('makemigrations', interactive=False)
call_command("migrate", interactive=False)

# db_name = settings.DATABASES['default']['NAME']
# db_user = settings.DATABASES['default']['USER']
# db_port = settings.DATABASES['default']['PORT']
# db_host = settings.DATABASES['default']['HOST']
# db_passwd = settings.DATABASES['default']['PASSWORD']
#
# utils.patch_db(db_name, db_user, db_port, db_host, db_passwd, settings.MONITORING_ENABLED)
except Exception as e:
logger.warning(f"Error while aligning the db: {e}", exc_info=e)

Expand Down Expand Up @@ -397,8 +364,15 @@ def execute_restore(self, **options):
err_cnt = 0

logger.info("*** Restoring GeoNode fixtures...")

fixtures_folder = os.path.join(target_folder, "fixtures")
if not os.path.exists(fixtures_folder):
# fixtures folder was introduced on 2024-02; make the restore command lenient about
# dumps created without such a folder (this behaviour may be removed in a short while)
fixtures_folder = target_folder

for app_name, dump_name in zip(config.app_names, config.dump_names):
fixture_file = os.path.join(target_folder, f"{dump_name}.json")
fixture_file = os.path.join(fixtures_folder, f"{dump_name}.json")

logger.info(f" - restoring '{fixture_file}'")
try:
Expand Down Expand Up @@ -430,109 +404,6 @@ def execute_restore(self, **options):
chmod_tree(media_root)
logger.info(f"Media files restored into '{media_root}'.")

# Restore Static Root
logger.info("*** Restore static root...")
if config.gs_data_dt_filter[0] is None:
shutil.rmtree(static_root, ignore_errors=True)

if not os.path.exists(static_root):
os.makedirs(static_root, exist_ok=True)

copy_tree(static_folder, static_root)
chmod_tree(static_root)
logger.info(f"Static root restored into '{static_root}'.")

# Restore Static Folders
logger.info("*** Restore static folders...")

for folder in static_folders:
logger.info(f"* Restoring {folder}...")

# skip restoration of static files of apps not located under PROJECT_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not folder.startswith(settings.PROJECT_ROOT):
logger.info(
f"Skipping static directory: {folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

if config.gs_data_dt_filter[0] is None:
logger.info(f"Cleaning {folder}...")
shutil.rmtree(folder, ignore_errors=True)

logger.info(f"Restoring {folder}...")
if not os.path.exists(folder):
os.makedirs(folder, exist_ok=True)

copy_tree(
os.path.join(static_files_folders, os.path.basename(os.path.normpath(folder))), folder
)
chmod_tree(folder)
logger.info(f"Static files restored into '{folder}'.")

# Restore Template Folders
logger.info("*** Restore template folders...")
for folder in template_folders:
logger.info(f"* Restoring {folder}...")

# skip restoration of template files of apps not located under PROJECT_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not folder.startswith(settings.PROJECT_ROOT):
logger.info(
f"Skipping template directory: {folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

if config.gs_data_dt_filter[0] is None:
logger.info(f"Cleaning {folder}...")
shutil.rmtree(folder, ignore_errors=True)

logger.info(f"Restoring {folder}...")
if not os.path.exists(folder):
os.makedirs(folder, exist_ok=True)

copy_tree(
os.path.join(template_files_folders, os.path.basename(os.path.normpath(folder))), folder
)
chmod_tree(folder)
logger.info(f"Template files restored into '{folder}'.")

# Restore Locale Folders
logger.info("*** Restore locale folders...")
for folder in locale_folders:
logger.info(f"* Restoring {folder}...")

# skip restoration of locale files of apps not located under PROJECT_ROOT path
# (check to prevent overriding files from site-packages
# in project-template based GeoNode projects)
if getattr(settings, "PROJECT_ROOT", None) and not folder.startswith(settings.PROJECT_ROOT):
logger.info(
f"Skipping locale directory: {folder}. "
f"It's not located under PROJECT_ROOT path: {settings.PROJECT_ROOT}."
)
continue

if config.gs_data_dt_filter[0] is None:
logger.info(f"Cleaning {folder}...")
shutil.rmtree(folder, ignore_errors=True)

logger.info(f"Restoring {folder}...")
if not os.path.exists(folder):
os.makedirs(folder, exist_ok=True)

copy_tree(
os.path.join(locale_files_folders, os.path.basename(os.path.normpath(folder))), folder
)
chmod_tree(folder)
logger.info(f"Locale Files Restored into '{folder}'.")

logger.info("*** Calling collectstatic...")
call_command("collectstatic", interactive=False)

# store backup info
restored_backup = RestoredBackup(
name=backup_file.rsplit("/", 1)[-1],
Expand Down Expand Up @@ -771,7 +642,7 @@ def bstr(x):

gs_bk_exec_status = gs_backup["restore"]["execution"]["status"]
gs_bk_exec_progress = gs_backup["restore"]["execution"]["progress"]
logger.info(f"Async backup status: {gs_bk_exec_status} - {gs_bk_exec_progress}")
logger.info(f"Async restore status: {gs_bk_exec_status} - {gs_bk_exec_progress}")
time.sleep(3)
else:
raise ValueError(error_backup.format(url, r.status_code, r.text))
Expand Down

0 comments on commit 3c66a54

Please sign in to comment.