reduce DB disk usage; better formatting of some values

Author: Tom Goetz

Fit

@@ -12,7 +12,7 @@
 
 from .garmindb import File, ActivitiesDb, Activities, ActivityRecords, ActivityLaps, ActivitySplits, ActivityClimbingSplits, ActivitiesDevices, ActivitiesDeviceUsed, \
     StepsActivities, SwimmingActivities,  CycleActivities, ClimbingActivities, PaddleActivities, ActivityLengths, ActivitySplitSummaries, ActivitiesBestEffort, \
-    Attributes
+    Attributes, ActivitiesEvaluations
 from .fit_file_processor import FitFileProcessor
 
 
@@ -82,12 +82,13 @@ def _write_activity_entry(self, fit_file, message_fields):
         self.activity_total_time = message_fields.get('total_timer_time')
 
     def _write_best_effort_entry(self, fit_file, message_fields):
+        root_logger.info("writing best_effort message %r for %s", message_fields, fit_file.filename)
         start_time = fit_file.utc_datetime_to_local(message_fields.start_time)
-        distance = message_fields.get('distance')
+        distance = message_fields.get('best_effort_distance')
         time = message_fields.get('time')
-        personal_record = message_fields.get('personal_record')
         if start_time and distance and time:
             sport = message_fields.get('sport')
+            personal_record = message_fields.get('personal_record')
             best_effort = {
                 'activity_id'       : self.activity_id,
                 'start_time'        : start_time,
@@ -96,12 +97,10 @@ def _write_best_effort_entry(self, fit_file, message_fields):
                 'time'              : time,
                 'personal_record'   : personal_record
             }
-            root_logger.info("writing best_effort %r -> %r for %s", message_fields, best_effort, fit_file.filename)
-            root_logger.info("writing best_effort distance %r", distance)
+            root_logger.info("writing best_effort %r for %s", best_effort, fit_file.filename)
             ActivitiesBestEffort.s_insert_or_update(self.garmin_act_db_session, best_effort, ignore_none=True, ignore_zero=True)
             if personal_record:
-                distance_int = int(distance)
-                self.__write_attribute(start_time, f'PR {distance_int} {sport.name}', str(time))
+                self.__write_attribute(start_time, f'PR {distance} {sport.name}', str(time))
 
     def _write_device_info_entry(self, fit_file, message_fields):
         device_serial_number = super()._write_device_info_entry(fit_file, message_fields)
@@ -475,6 +474,7 @@ def _write_session_entry(self, fit_file, message_fields):
             'cycles'                            : message_fields.get('total_cycles'),
             'laps'                              : message_fields.get('num_laps'),
             'training_effect'                   : message_fields.get('total_training_effect'),
+            'training_load'                     : message_fields.get('training_load'),
             'anaerobic_training_effect'         : message_fields.get('total_anaerobic_training_effect'),
             'primary_benefit'                   : message_fields.get('primary_benefit'),
         }
@@ -509,6 +509,15 @@ def _write_session_entry(self, fit_file, message_fields):
                     root_logger.warning("No sport handler for type %s from %s: %s", sport, fit_file.filename, message_fields)
             except Exception as e:
                 root_logger.error("Exception in %s from %s: %s", function_name, fit_file.filename, e)
+        feel = message_fields.get('workout_feel')
+        effort = message_fields.get('workout_rpe')
+        if feel or effort:
+            evuation = {
+                'activity_id'       : activity_id,
+                'self_eval_feel'    : feel,
+                'self_eval_effort'  : effort
+            }
+            ActivitiesEvaluations.s_find_or_create(self.garmin_act_db_session, evuation)
 
     def _write_time_in_zone_entry(self, fit_file, message_fields):
         """Write hz zones message to the database."""

Plugins

@@ -15,6 +15,7 @@
 class ActivityFitPluginBase(PluginBase):
     """Base class for GarminDb activity FIT file plugins that handle data based on ids of applications or developer fields, sport or sub-sport ids, etc."""
 
+    __plugin_interface_version__ = 1
     _type = 'ActivityFit'
 
     @classmethod
@@ -36,6 +37,10 @@ def matches_activity_file(cls, fit_file):
     def init_activity(cls, act_db_class, activities_table):
         """Initialize an instance of the plugin as an activity FIT file plugin."""
         logger.info("Initializing tables for activity plugin %s with activities table %s", cls.__name__, activities_table)
+        plugin_interface_implementation_version = getattr(cls, '__plugin_interface_implementation_version__', 0)
+        if plugin_interface_implementation_version != cls.__plugin_interface_version__:
+            raise Exception(f'Error loading plugin {cls.__name__} version mismatch: '
+                            + f'{cls.__plugin_interface_version__} vs {plugin_interface_implementation_version} - update plugin!')
         if hasattr(cls, '_records_tablename') and 'record' not in cls._tables:
             cls._tables['record'] = activities_table.create(cls._records_tablename, act_db_class, cls._records_version, cls._records_pk, cls._records_cols)
         if hasattr(cls, '_laps_tablename') and 'lap' not in cls._tables:

garmindb/activity_fit_file_processor.py

@@ -13,7 +13,7 @@
 from idbutils import JsonFileProcessor
 
 from .garmin_connect_enums import Event, get_summary_sport, get_details_sport
-from .garmindb import ActivitiesDb, Activities, StepsActivities, PaddleActivities, CycleActivities
+from .garmindb import ActivitiesDb, Activities, ActivitiesCourses, ActivitiesEvaluations, StepsActivities, PaddleActivities, CycleActivities
 
 
 logger = logging.getLogger(__file__)
@@ -64,20 +64,20 @@ def _process_common(self, json_data):
             'stop_long'                 : self._get_field(json_data, 'endLongitude', float),
             'distance'                  : distance.kms_or_miles(self.measurement_system),
             'laps'                      : self._get_field(json_data, 'lapCount'),
-            'avg_hr'                    : self._get_field(json_data, 'averageHR', float),
-            'max_hr'                    : self._get_field(json_data, 'maxHR', float),
-            'calories'                  : self._get_field(json_data, 'calories', float),
+            'avg_hr'                    : self._get_field(json_data, 'averageHR', float, 1),
+            'max_hr'                    : self._get_field(json_data, 'maxHR', float, 1),
+            'calories'                  : self._get_field(json_data, 'calories', int),
             'avg_speed'                 : avg_speed.kph_or_mph(self.measurement_system) if avg_speed is not None else None,
             'max_speed'                 : max_speed.kph_or_mph(self.measurement_system) if max_speed is not None else None,
             'ascent'                    : ascent.meters_or_feet(self.measurement_system) if ascent is not None else None,
             'descent'                   : descent.meters_or_feet(self.measurement_system) if descent is not None else None,
             'max_temperature'           : max_temperature.c_or_f(self.measurement_system) if max_temperature is not None else None,
             'min_temperature'           : min_temperature.c_or_f(self.measurement_system) if min_temperature is not None else None,
             'avg_temperature'           : avg_temperature.c_or_f(self.measurement_system) if avg_temperature is not None else None,
-            'training_effect'           : self._get_field(json_data, 'aerobicTrainingEffect', float),
-            'anaerobic_training_effect' : self._get_field(json_data, 'anaerobicTrainingEffect', float),
-            'max_rr'                    : self._get_field(json_data, 'maxRespirationRate', float),
-            'avg_rr'                    : self._get_field(json_data, 'avgRespirationRate', float),
+            'training_effect'           : self._get_field(json_data, 'aerobicTrainingEffect', float, 1),
+            'anaerobic_training_effect' : self._get_field(json_data, 'anaerobicTrainingEffect', float, 1),
+            'max_rr'                    : self._get_field(json_data, 'maxRespirationRate', float, 1),
+            'avg_rr'                    : self._get_field(json_data, 'avgRespirationRate', float, 1),
         }
 
     def _process_json(self, json_data):
@@ -114,9 +114,9 @@ def _process_steps_activity(self, activity_id, activity_summary):
             'steps'                     : self._get_field(activity_summary, 'steps', float),
             'avg_steps_per_min'         : self._get_field(activity_summary, 'averageRunningCadenceInStepsPerMinute', float),
             'max_steps_per_min'         : self._get_field(activity_summary, 'maxRunningCadenceInStepsPerMinute', float),
-            'avg_step_length'           : avg_step_length.meters_or_feet(self.measurement_system),
+            'avg_step_length'           : avg_step_length.meters_or_feet(self.measurement_system) if avg_step_length else None,
             'avg_gct_balance'           : self._get_field(activity_summary, 'avgGroundContactBalance', float),
-            'avg_vertical_oscillation'  : avg_vertical_oscillation.meters_or_feet(self.measurement_system),
+            'avg_vertical_oscillation'  : avg_vertical_oscillation.meters_or_feet(self.measurement_system) if avg_vertical_oscillation else None,
             'avg_ground_contact_time'   : fitfile.conversions.ms_to_dt_time(self._get_field(activity_summary, 'avgGroundContactTime', float)),
             'vo2_max'                   : self._get_field(activity_summary, 'vO2MaxValue', float),
         }
@@ -291,39 +291,32 @@ def _process_fitness_equipment(self, sub_sport, activity_id, json_data):
         root_logger.debug("fitness_equipment (%s) for %d: %r", sub_sport, activity_id, json_data)
         self._call_process_func(sub_sport.name, None, activity_id, json_data)
 
-    @classmethod
-    def get_self_eval_feel(cls, value):
-        """Return the Garmin Connect self evaluation 'How did you feel' label for the activity."""
-        levels = [(100, "Very Strong"), (75, "Strong"), (50, "Normal"), (25, "Weak"), (0, "Very Weak")]
-        for threshold, label in levels:
-            if value >= threshold:
-                return label
-
-    @classmethod
-    def get_self_eval_effort(cls, value):
-        """Return the Garmin Connect self evaluation perceived effort label for the activity."""
-        levels = [(100, "Maximum"), (90, "Extremely Hard"), (70, "Very Hard"), (50, "Hard"),
-                  (40, "Somewhat Hard"), (30, "Moderate"), (20, "Light"), (10, "Very Light"), (0, "None")]
-        for threshold, label in levels:
-            if value >= threshold:
-                return label
-
     def _activities_process_json(self, json_data):
         activity_id = json_data['activityId']
         metadata_dto = json_data['metadataDTO']
         summary_dto = json_data['summaryDTO']
         sport, sub_sport = get_details_sport(json_data)
-        self_eval_feel = self._get_field(summary_dto, 'directWorkoutFeel', int)
-        self_eval_effort = self._get_field(summary_dto, 'directWorkoutRpe', int)
         activity = {
             'activity_id'           : activity_id,
-            'course_id'             : self._get_field(metadata_dto, 'associatedCourseId', int),
-            'device_serial_number'  : self._get_field(metadata_dto, 'deviceId', int),
-            'self_eval_feel'        : self.get_self_eval_feel(self_eval_feel) if (self_eval_feel is not None) else None,
-            'self_eval_effort'      : self.get_self_eval_effort(self_eval_effort) if (self_eval_effort is not None) else None,
-            'training_load'         : self._get_field(summary_dto, 'activityTrainingLoad', float)
+            'training_load'         : self._get_field(summary_dto, 'activityTrainingLoad', float, 1)
         }
         activity.update(self._process_common(summary_dto))
         Activities.s_insert_or_update(self.garmin_act_db_session, activity, ignore_none=True)
+        course_id = self._get_field(metadata_dto, 'associatedCourseId', int)
+        if course_id:
+            activity_course = {
+                'activity_id'   : activity_id,
+                'course_id'     : course_id,
+            }
+            ActivitiesCourses.s_insert_or_update(self.garmin_act_db_session, activity_course, ignore_none=True)
+        self_eval_feel = self._get_field(summary_dto, 'directWorkoutFeel', int)
+        self_eval_effort = self._get_field(summary_dto, 'directWorkoutRpe', int)
+        if self_eval_feel or self_eval_effort:
+            evuation = {
+                'activity_id'       : activity_id,
+                'self_eval_feel'    : fitfile.fields.ActivityEvalFeel.get_self_eval_feel(self_eval_feel) if (self_eval_feel is not None) else None,
+                'self_eval_effort'  : fitfile.fields.ActivityEvalEffort.get_self_eval_effort(self_eval_effort) if (self_eval_effort is not None) else None
+            }
+            ActivitiesEvaluations.s_insert_or_update(self.garmin_act_db_session, evuation)
         self._call_process_func(sport.name, sub_sport, activity_id, json_data)
         return 1

garmindb/activity_fit_plugin_base.py

@@ -12,5 +12,6 @@
 from .sleep_db import SleepDb, SleepEvents, SleepAssessments, Sleep, Naps
 from .hrv_db import HrvDb, HrvValue, HrvStatusSummary
 from .activities_db import ActivitiesDb, Activities, ActivityLaps, ActivityRecords, ActivitiesDevices, ActivitiesDeviceUsed, ActivitySplits, ActivityClimbingSplits, \
-    SportActivities, StepsActivities, PaddleActivities, CycleActivities, ClimbingActivities, ActivityLengths, ActivitySplitSummaries, SwimmingActivities, ActivitiesBestEffort
+    SportActivities, StepsActivities, PaddleActivities, CycleActivities, ClimbingActivities, ActivityLengths, ActivitySplitSummaries, SwimmingActivities, ActivitiesBestEffort, \
+    ActivitiesCourses, ActivitiesEvaluations
 from .garmin_summary_db import GarminSummaryDb, Summary, YearsSummary, MonthsSummary, WeeksSummary, DaysSummary, IntensityHR

garmindb/garmin_json_data.py

@@ -249,7 +249,6 @@ class ActivityClimbingSplits(ActivitiesDb.Base, idbutils.DbObject):
 
     activity_id = Column(String, ForeignKey('activities.activity_id'))
     split = Column(Integer)
-
     grade = Column(String)          # climbing grade
     completed = Column(Boolean)     # climbing route
     falls = Column(Integer)         # climbing number of falls
@@ -323,16 +322,9 @@ class Activities(ActivitiesDb.Base, ActivitiesLapsCommon):
     name = Column(String)
     description = Column(String)
     type = Column(String)
-    course_id = Column(Integer)
     laps = Column(Integer)
     sport = Column(String)
     sub_sport = Column(String)
-
-    device_serial_number = Column(BigInteger)
-
-    self_eval_feel = Column(String)
-    self_eval_effort = Column(String)
-
     training_load = Column(Float)
     training_effect = Column(Float)
     anaerobic_training_effect = Column(Float)
@@ -510,7 +502,7 @@ class ActivitiesBestEffort(ActivitiesDb.Base, idbutils.DbObject):
     activity_id = Column(String, ForeignKey('activities.activity_id'), primary_key=True)
     start_time = Column(DateTime)
     sport = Column(Enum(fitfile.fields.Sport))
-    distance = Column(Float)
+    distance = Column(String)
     time = Column(Time, nullable=False, default=datetime.time.min)
     personal_record = Column(Boolean)
 
@@ -526,6 +518,30 @@ def get_activity(cls, db, activity_id):
             return cls.s_get_activity(session, activity_id)
 
 
+class ActivitiesCourses(ActivitiesDb.Base, idbutils.DbObject):
+    """Class represents a database table that maps activities to course ids."""
+
+    __tablename__ = 'activities_courses'
+    __get_col_name__ = "activity_id"
+
+    db = ActivitiesDb
+    table_version = 2
+
+    activity_id = Column(String, primary_key=True)
+    course_id = Column(Integer)
+
+    @classmethod
+    def s_get_activity(cls, session, activity_id):
+        """Return all activity devices records for a given activity_id."""
+        return session.query(cls).filter(cls.activity_id == activity_id).all()
+
+    @classmethod
+    def get_activity(cls, db, activity_id):
+        """Return all activity devices records for a given activity_id."""
+        with db.managed_session() as session:
+            return cls.s_get_activity(session, activity_id)
+
+
 class ActivitiesDevices(ActivitiesDb.Base, idbutils.DbObject):
     """Class represents a database table that maps device ids to activities (by id) that they were used in."""
 
@@ -552,6 +568,31 @@ def get_activity(cls, db, activity_id):
             return cls.s_get_activity(session, activity_id)
 
 
+class ActivitiesEvaluations(ActivitiesDb.Base, idbutils.DbObject):
+    """Class represents a database table that that stores self evaluations for activities."""
+
+    __tablename__ = 'activities_evaluations'
+    __get_col_name__ = "activity_id"
+
+    db = ActivitiesDb
+    table_version = 1
+
+    activity_id = Column(String, ForeignKey('activities.activity_id'), primary_key=True)
+    self_eval_feel = Column(String)
+    self_eval_effort = Column(String)
+
+    @classmethod
+    def s_get_activity(cls, session, activity_id):
+        """Return all activity devices records for a given activity_id."""
+        return session.query(cls).filter(cls.activity_id == activity_id).all()
+
+    @classmethod
+    def get_activity(cls, db, activity_id):
+        """Return all activity devices records for a given activity_id."""
+        with db.managed_session() as session:
+            return cls.s_get_activity(session, activity_id)
+
+
 class ActivitiesDeviceUsed(ActivitiesDb.Base, idbutils.DbObject):
     """Class represents a database table that show which devices were used for metrics during activities (by serial number)."""
 
@@ -598,20 +639,20 @@ def _create_activity_view(cls, db, selectable):
         """Create a database view for a activity type."""
         view_name = cls._get_default_view_name()
         logger.debug("Creating activity view %s if needed.", view_name)
-        cls.create_join_view(db, view_name, selectable, Activities, order_by=Activities.start_time.desc())
+        cls.create_join_view(db, view_name, selectable, [Activities], order_by=Activities.start_time.desc())
 
     @classmethod
     def _create_sport_view(cls, db, selectable, sport):
         """Create a database view for a sport based activity type."""
         # SQL equality is `=` and string literals use single quotes. sqlite
         # tolerates `==` and double-quoted literals; postgres does not.
         filter = literal_column(f"{Activities.sport} = '{sport}'")
-        cls.create_join_view(db, f'{sport}_activities_view', selectable, Activities, filter, Activities.start_time.desc())
+        cls.create_join_view(db, f'{sport}_activities_view', selectable, [Activities], filter, Activities.start_time.desc())
 
     @classmethod
     def _create_course_view(cls, db, selectable, course_id):
-        filter = literal_column(f'{Activities.course_id} = {course_id}')
-        cls.create_join_view(db, f'course_{course_id}_view', selectable, Activities, filter, Activities.start_time.desc())
+        filter = literal_column(f'{ActivitiesCourses.course_id} = {course_id}')
+        cls.create_join_view(db, f'course_{course_id}_view', selectable, [Activities, ActivitiesCourses], filter, Activities.start_time.desc())
 
     @classmethod
     def create_view(cls, db):
@@ -669,7 +710,7 @@ def _view_selectable(cls, include_sport=False, include_subsport=False, include_t
         if include_type:
             selectable.append(Activities.type.label('type'))
         if include_course:
-            selectable.append(Activities.course_id.label('course_id'))
+            selectable.append(ActivitiesCourses.course_id.label('course_id'))
         selectable += [
             Activities.start_time.label('start_time'),
             Activities.stop_time.label('stop_time'),