wip agu 2024, small bugfix

doc/how_tos.rst

@@ -56,6 +56,7 @@ This recipe is a kind of "let's check this data rapidly":
     msnoise cc stack -m
     msnoise cc dvv compute_mwcs
     msnoise cc dvv compute_dtt
+    msnoise cc dvv compute_dvv
     msnoise cc dvv plot dvv
 
 

msnoise/api.py

@@ -838,9 +838,11 @@ def massive_insert_job(jobs):
     :param jobs: a list of :class:`~msnoise.msnoise_table_def.declare_tables.Job` to insert.
     """
     engine = get_engine()
-    engine.execute(
-        Job.__table__.insert(),
-        jobs)
+    with engine.connect() as conn:
+        conn.execute(
+            Job.__table__.insert(),
+            jobs)
+        conn.commit()
 
 
 def massive_update_job(session, jobs, flag="D"):
@@ -2365,6 +2367,7 @@ def compute_dvv(session, filterid, mov_stack, pairs=None, components=None, param
                 dtt = xr_get_dtt(s1, s2, comp, filterid, mov_stack)
                 all.append(dtt)
             except FileNotFoundError:
+                traceback.print_exc()
                 continue
     if not len(all):
         raise ValueError

msnoise/s07_compute_dvv.py

@@ -1,3 +1,4 @@
+import traceback
 
 import matplotlib.gridspec as gridspec
 import matplotlib.pyplot as plt
@@ -28,6 +29,7 @@ def main(interval=1, loglevel="INFO"):
                     dvv = compute_dvv(db, filterid, mov_stack, pairs=None,
                                   components=components, params=params)
                 except ValueError:
+                    traceback.print_exc()
                     logger.error("No data for f%i m%i: %s" % (filterid, mov_stack, components))
                     continue
                 xr_save_dvv(components, filterid, mov_stack, dvv)

msnoise/scripts/msnoise.py

@@ -330,7 +330,7 @@ def db_execute(ctx, sql_command, outfile=None, show=True):
         if not len(cmd):
             continue
         logger.info("Executing '%s'" % cmd)
-        r = db.execute(cmd)
+        r = db.execute(text(cmd))
         if cmd.count("select") or cmd.count("SELECT"):
             result = r.fetchall()
             if not len(result):
@@ -374,26 +374,26 @@ def db_upgrade():
             # print("Passing %s: already in DB" % name)
             continue
     try:
-        db.execute("CREATE UNIQUE INDEX job_index ON %sjobs (day, pair, "
+        db.execute(text("CREATE UNIQUE INDEX job_index ON %sjobs (day, pair, "
                    "jobtype)" %
-                   prefix)
+                   prefix))
         db.commit()
     except:
         logging.info("It looks like the v1.5 'job_index' is already in the DB")
         db.rollback()
 
     try:
-        db.execute("CREATE INDEX job_index2 ON %sjobs (jobtype, flag)" %
-                   prefix)
+        db.execute(text("CREATE INDEX job_index2 ON %sjobs (jobtype, flag)" %
+                   prefix))
         db.commit()
     except:
         logging.info("It looks like the v1.6 'job_index2' is already in the DB")
         db.rollback()
 
     try:
-        db.execute("CREATE UNIQUE INDEX da_index ON %sdata_availability (path, "
+        db.execute(text("CREATE UNIQUE INDEX da_index ON %sdata_availability (path, "
                    "file, net, sta, loc, chan)" %
-                   prefix)
+                   prefix))
         db.commit()
     except:
         logging.info("It looks like the v1.5 'da_index' is already in the DB")
@@ -419,7 +419,7 @@ def db_clean_duplicates():
                 'WHERE (j1.ref > j2.ref) AND (j1.day=j2.day) '\
                 'AND (j1.pair=j2.pair) AND (j1.jobtype=j2.jobtype)'\
                 .format(prefix)
-    db.execute(query)
+    db.execute(text(query))
     db.commit()
     db.close()
 
@@ -440,8 +440,9 @@ def db_dump(format):
         meta.reflect(bind=engine)
 
         for table in meta.sorted_tables:
-            r = [dict(row) for row in engine.execute(table.select())]
-            df = pd.DataFrame(r)
+            with engine.connect() as conn:
+                rows = conn.execute(table.select()).all()
+            df = pd.DataFrame(rows)
             logger.info("Dumping table %s to %s.csv" % (table.name, table.name))
             df.to_csv("%s.csv" % table.name, index=False)
     else:
@@ -591,8 +592,8 @@ def reset(jobtype, all, rule):
     prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
     session = connect()
     if jobtype == "DA":
-        session.execute("UPDATE {0}data_availability SET flag='M'"
-                        .format(prefix))
+        session.execute(text("UPDATE {0}data_availability SET flag='M'"
+                        .format(prefix)))
     elif jobtype != jobtype.upper():
         logging.info("The jobtype %s is not uppercase (usually jobtypes"
                      " are uppercase...)"%jobtype)
@@ -727,10 +728,10 @@ def new_jobs(init, nocc, hpc=""):
         prefix = (dbini.prefix + '_') if dbini.prefix != '' else ''
         left, right = hpc.split(':')
         db = connect()
-        db.execute("INSERT INTO {prefix}jobs (pair, day, jobtype, flag) "
+        db.execute(text("INSERT INTO {prefix}jobs (pair, day, jobtype, flag) "
                    "SELECT pair, day, '{right_type}', 'T' FROM {prefix}jobs "
                    "WHERE jobtype='{left_type}' AND flag='D';"
-                   .format(prefix=prefix, right_type=right, left_type=left))
+                   .format(prefix=prefix, right_type=right, left_type=left)))
         db.commit()
         db.close()
 

msnoise/templates/admin/index.html

@@ -78,6 +78,7 @@
     get_num_jobs('STACK', generic_update_jobs);
     get_num_jobs('MWCS', generic_update_jobs);
     get_num_jobs('DTT', generic_update_jobs);
+    get_num_jobs('DVV', generic_update_jobs);
     data_availability_flags(generic_update_jobs);
 
    }
@@ -288,9 +289,9 @@ <h3>MWCS Jobs</h3>
              <button id='reset_MWCS' value='MWCS' class="btn
   btn-success btn-xs">Reset MWCS jobs "in progress"</button>
 
-     <button id='reset_MWCS_all' value='Reset CC' class="btn
+     <button id='reset_MWCS_all' value='Reset MWCS' class="btn
   btn-warning btn-xs">Reset <b>all</b> MWCS jobs</button>
-<button id='delete_MWCS_all' value='Reset STACK' class="btn
+<button id='delete_MWCS_all' value='Reset MWCS' class="btn
   btn-danger btn-xs">Delete <b>all</b> MWCS jobs</button>
   <div class="alert fade in" id="reset_MWCS_console_div" style="display:none;width:300px;">
     <button type="button" class="close">x</button>
@@ -308,9 +309,9 @@ <h3>Relative Travel Time Jobs</h3>
              <button id='reset_DTT' value='DTT' class="btn
   btn-success btn-xs">Reset DTT jobs "in progress"</button>
 
-     <button id='reset_DTT_all' value='Reset CC' class="btn
+     <button id='reset_DTT_all' value='Reset DTT' class="btn
   btn-warning btn-xs">Reset <b>all</b> DTT jobs</button>
-<button id='delete_DTT_all' value='Reset STACK' class="btn
+<button id='delete_DTT_all' value='Reset DTT' class="btn
   btn-danger btn-xs">Delete <b>all</b> DTT jobs</button>
   <div class="alert fade in" id="reset_DTT_console_div" style="display:none;width:300px;">
     <button type="button" class="close">x</button>
@@ -321,6 +322,27 @@ <h3>Relative Travel Time Jobs</h3>
 <div class="progress">
   <div id='DTT' class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width: 0%;">0%</div>
 </div>
+
+      <h3>DV/V jobs</h3>
+             <button id='reset_DVV' value='DVV' class="btn
+  btn-success btn-xs">Reset DVV jobs "in progress"</button>
+
+     <button id='reset_DVV_all' value='Reset DVV' class="btn
+  btn-warning btn-xs">Reset <b>all</b> DVV jobs</button>
+<button id='delete_DVV_all' value='Reset DVV' class="btn
+  btn-danger btn-xs">Delete <b>all</b> DVV jobs</button>
+  <div class="alert fade in" id="reset_DVV_console_div" style="display:none;width:300px;">
+    <button type="button" class="close">x</button>
+    <div id='reset_DVV_console'></div>
+  </div>
+    <br>
+<span id='jobs_DVV'></span>
+<div class="progress">
+  <div id='DVV' class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width: 0%;">0%</div>
+</div>
+
+
+
   </div>
 </div>
 
@@ -354,3 +376,4 @@ <h2>Actions ?</h2>
 </script>
 {% endblock body %}
 
+