Feature/resync hdx

api/urls.py

@@ -6,7 +6,7 @@
 
 from .views import (ConfigurationViewSet, ExportRunViewSet,
                     HDXExportRegionViewSet, PartnerExportRegionViewSet, JobViewSet, permalink, get_overpass_timestamp,
-                    cancel_run, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status)
+                    cancel_run,sync_to_hdx_api, get_user_permissions, request_geonames, get_overpass_status, get_groups, stats, run_stats, request_nominatim,machine_status)
 
 router = DefaultRouter(trailing_slash=False)
 router.register(r'jobs', JobViewSet, base_name='jobs')
@@ -36,6 +36,8 @@
     url(r'^stats$', stats),
     url(r'^run_stats$', run_stats),
     url(r'^status$', machine_status),
+    url(r'^sync_to_hdx_api$', sync_to_hdx_api),
     url(r'^cancel_run$', cancel_run),
 
+
 ]

api/views.py

@@ -10,6 +10,7 @@
 from collections import Counter
 import os
 import io
+import pickle
 import csv
 import dateutil.parser
 import requests
@@ -21,6 +22,7 @@
 from django.contrib.auth.models import Group
 from django.contrib.gis.geos import GEOSGeometry, Polygon
 from django.db.models import Q
+from django.core.exceptions import ObjectDoesNotExist
 from django.http import (
     JsonResponse,
     HttpResponse,
@@ -55,6 +57,8 @@
 from hdx_exports.hdx_export_set import sync_region
 from rtree import index
 import psutil
+from asgiref.sync import async_to_sync
+import asyncio
 
 # Get an instance of a logger
 LOG = logging.getLogger(__name__)
@@ -650,7 +654,7 @@ def request_geonames(request):
 
             if str(keyword).lower().startswith('osm'):
                 lst=keyword.split(" ")
-                if len(lst)>=1:
+                if len(lst)>1:
                     keyword=lst[1]
                     try : 
                         osm_id= int(keyword) 
@@ -680,7 +684,7 @@ def request_geonames(request):
 
         if str(keyword).lower().startswith('tm'):
                 lst=keyword.split(" ")
-                if len(lst)>=1:
+                if len(lst)>1:
 
                     keyword=lst[1]
                     if tm_url:
@@ -773,6 +777,105 @@ def get_groups(request):
     return JsonResponse({"groups": groups})
 
 
+async def sync_to_hdx_api_async(run_uid):
+    try:
+        run = ExportRun.objects.get(uid=run_uid)
+    except ExportRun.DoesNotExist:
+        return {"error": "Invalid run UID"}
+
+    try:
+        region = HDXExportRegion.objects.get(job_id=run.job_id)
+    except HDXExportRegion.DoesNotExist:
+        return JsonResponse({"error": "HDXExportRegion not found"}, status=404)
+
+    public_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
+    LOG.debug(public_dir)
+    pickle_file_path = os.path.join(public_dir, 'all_zips.pkl')
+
+    if os.path.exists(pickle_file_path):
+        try:
+            with open(pickle_file_path, 'rb') as file:
+                all_zips_data = file.read()
+            all_zips = pickle.loads(all_zips_data)
+            LOG.debug("Calling hdx API")
+            sync_region(region, all_zips, public_dir)
+            run.hdx_sync_status = True
+        except Exception as ex:
+            run.sync_status = False
+            LOG.error(ex)
+            return JsonResponse({"error": "Sync failed"}, status=500)
+    else:
+        return JsonResponse({"error": "No exports available"}, status=404)
+
+    run.save()
+    LOG.debug('Sync Success to HDX for run: {0}'.format(run_uid))
+
+
+    return {"success": "Sync to HDX completed successfully"}
+
+
+@require_http_methods(["GET"])
+@async_to_sync
+async def sync_to_hdx_api(request):
+    if not request.user.is_superuser:
+        return JsonResponse({"error": "Permission denied"}, status=403)
+
+    run_uid = request.GET.get("run_uid")
+    if run_uid:
+        # loop = asyncio.get_event_loop()
+        # loop.create_task(sync_to_hdx_api_async(run_uid))
+        await sync_to_hdx_api_async(run_uid)
+        # asyncio.run(sync_to_hdx_api_async(run_uid))
+        return JsonResponse({"message": "Sync request received and is being processed in the background."}, status=202)
+
+    return JsonResponse({"error": "Missing run UID"}, status=400)
+
+# @require_http_methods(["GET"])
+# def sync_to_hdx_api(request):
+#     if not request.user.is_superuser:
+#         return HttpResponseForbidden()
+
+#     run_uid = request.GET.get("run_uid")
+#     if run_uid:
+#         LOG.debug('Syncing to HDX for run: {0}'.format(run_uid))
+#         try:
+#             run = ExportRun.objects.get(uid=run_uid)
+#         except ExportRun.DoesNotExist:
+#             return JsonResponse({"error": "Invalid run UID"}, status=404)
+
+#         try:
+#             region = HDXExportRegion.objects.get(job_id=run.job_id)
+#         except HDXExportRegion.DoesNotExist:
+#             return JsonResponse({"error": "HDXExportRegion not found"}, status=404)
+
+#         public_dir = os.path.join(settings.EXPORT_DOWNLOAD_ROOT, run_uid)
+#         LOG.debug(public_dir)
+#         pickle_file_path = os.path.join(public_dir, 'all_zips.pkl')
+
+#         if os.path.exists(pickle_file_path):
+#             try:
+#                 with open(pickle_file_path, 'rb') as file:
+#                     all_zips_data = file.read()
+#                 all_zips = pickle.loads(all_zips_data)
+#                 LOG.debug("Calling hdx API")
+#                 sync_region(region, all_zips, public_dir)
+#                 run.hdx_sync_status = True
+#             except Exception as ex:
+#                 run.sync_status = False
+#                 LOG.error(ex)
+#                 return JsonResponse({"error": "Sync failed"}, status=500)
+#         else:
+#             return JsonResponse({"error": "No exports available"}, status=404)
+
+#         run.save()
+#         LOG.debug('Sync Success to HDX for run: {0}'.format(run_uid))
+
+#         return JsonResponse({"success": "Sync to HDX completed successfully"}, status=200)
+
+#     return JsonResponse({"error": "Missing run UID"}, status=400)
+
+
+
 from dramatiq_abort import abort
 
 

hdx_exports/hdx_export_set.py

@@ -3,6 +3,7 @@
 import django.utils.text
 from hdx.data.dataset import Dataset
 from osm_export_tool.mapping import Mapping
+import logging 
 
 FILTER_CRITERIA = """
 This theme includes all OpenStreetMap features in this area matching:
@@ -27,16 +28,21 @@ def slugify(str):
     s = django.utils.text.slugify(str)
     return s.replace('-','_')
 
-def sync_datasets(datasets,update_dataset_date=False):
+def sync_datasets(datasets, update_dataset_date=False):
     for dataset in datasets:
-        exists = Dataset.read_from_hdx(dataset['name'])
-        if exists:
-            if update_dataset_date:
+        try:
+            exists = Dataset.read_from_hdx(dataset['name'])
+            if exists:
+                if update_dataset_date:
+                    dataset.set_date_of_dataset(datetime.now())
+                dataset.update_in_hdx()
+            else:
                 dataset.set_date_of_dataset(datetime.now())
-            dataset.update_in_hdx()
-        else:
-            dataset.set_date_of_dataset(datetime.now())
-            dataset.create_in_hdx(allow_no_resources=True)
+                dataset.create_in_hdx(allow_no_resources=True)
+        except Exception as e:
+            # Log the error and continue with the next dataset
+            logging.error(f"Error syncing dataset {dataset['name']}: {str(e)}")
+
 
 def sync_region(region,files=[],public_dir=''):
     export_set = HDXExportSet(

tasks/task_runners.py

@@ -8,6 +8,7 @@
 import shutil
 import zipfile
 import traceback
+import pickle
 
 import django
 from dramatiq.middleware import TimeLimitExceeded
@@ -488,8 +489,12 @@ def add_metadata(z,theme):
         if settings.SYNC_TO_HDX:
             LOG.debug('Syncing to HDX for run: {0}'.format(run_uid))
             region = HDXExportRegion.objects.get(job_id=run.job_id)
+            all_zips_data = pickle.dumps(all_zips)
+            public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid)
+            pickle_file_path = os.path.join(download_dir,'all_zips.pkl')
+            with open(pickle_file_path, 'wb') as file:
+                file.write(all_zips_data)
             try:
-                public_dir = settings.HOSTNAME + join(settings.EXPORT_MEDIA_ROOT, run_uid)
                 sync_region(region,all_zips,public_dir)
                 run.hdx_sync_status = True
             except Exception as ex:

ui/app/components/ExportDetails.js

@@ -225,7 +225,7 @@ class ExportRuns extends Component {
                       </Alert>
                     </td>
                   </tr>
-                  { (run.status === "SUBMITTED" || run.status == "RUNNING") ?(
+                  { (run.status === "SUBMITTED" || run.status === "RUNNING") ?(
                   <RequirePermission required={["auth.add_user"]}>
                     <tr>
                       <td>
@@ -273,7 +273,34 @@ class ExportRuns extends Component {
                         />
                       </td>
                       <td colSpan="3">
-                      {run.hdx_sync_status ? "Uploaded" : "Not Uploaded"}
+                      {run.hdx_sync_status ? "Uploaded " : "Not Uploaded "}
+
+                        <Button
+                          bsStyle="success"
+                          disabled={run.hdx_sync_status}
+                          onClick={async () => {
+                            try {
+                              const token = selectAuthToken(this.props.state_token);
+                              const response = await axios({
+                                baseURL: window.EXPORTS_API_URL,
+                                headers: {
+                                  Authorization: `Bearer ${token}`
+                                },
+                                method: "GET",
+                                url: `/api/sync_to_hdx_api`,
+                                params: {
+                                  run_uid: run.uid
+                                }
+                              });
+                              alert(response.data.message);
+                            } catch (err) {
+                              console.warn(err);
+                              alert(err);
+                            }
+                          }}
+                        >
+                          <FormattedMessage id="ui.resync_hdx" defaultMessage="Resync" />
+                        </Button>
                       </td>
                     </tr>
                   </RequirePermission>