# Module `Gnumed.business.gmIncomingData`

Handling of area.

## Functions

`def create_incoming_data(data_type: str = None, filename: str = None, verify_import: bool = False) ‑> cIncomingData`
  Expand source code

      def create_incoming_data(data_type:str=None, filename:str=None, verify_import:bool=False) -> cIncomingData:
              conn = gmPG2.get_connection(readonly = False)
              args = {'typ': data_type}
              SQL = """
                      INSERT INTO clin.incoming_data (type, data)
                      VALUES (%(typ)s, 'new data'::bytea)
                      RETURNING pk"""
              rows = gmPG2.run_rw_query (
                      link_obj = conn,
                      end_tx = False,
                      sql = SQL,
                      args = args,
                      return_data = True
              )
              pk = rows[0]['pk']
              incoming = cIncomingData(aPK_obj = pk, link_obj = conn)
              if incoming.update_data_from_file(fname = filename, link_obj = conn, verify_import = verify_import):
                      conn.commit()
                      return incoming

              conn.rollback()
              _log.debug('cannot update incoming_data stub from file, rolled back')
              return None

`def data_exists(filename: str) ‑> bool`
  Expand source code

      def data_exists(filename:str) -> bool:
              """Check by md5 hash whether data in filename already in database."""
              local_md5 = gmTools.file2md5(filename = filename)
              SQL = 'SELECT EXISTS(SELECT 1 FROM clin.incoming_data WHERE md5(data) = %(local_md5)s) AS data_exists'
              args = {'local_md5': local_md5}
              rows = gmPG2.run_ro_query(sql = SQL, args = args)
              return rows[0]['data_exists']

  Check by md5 hash whether data in filename already in database.

`def delete_incoming_data(pk_incoming_data=None)`
  Expand source code

      def delete_incoming_data(pk_incoming_data=None):
              SQL = "DELETE FROM clin.incoming_data WHERE pk = %(pk)s"
              args = {'pk': pk_incoming_data}
              gmPG2.run_rw_query(sql = SQL, args = args)
              return True

`def get_incoming_data(order_by=None, return_pks=False)`
  Expand source code

      def get_incoming_data(order_by=None, return_pks=False):
              if order_by is None:
                      order_by = 'true'
              else:
                      order_by = 'true ORDER BY %s' % order_by
              SQL = _SQL_get_incoming_data % order_by
              rows = gmPG2.run_ro_query(sql = SQL)
              if return_pks:
                      return [ r['pk_incoming_data'] for r in rows ]

              return [ cIncomingData(row = {'data': r, 'pk_field': 'pk_incoming_data'}) for r in rows ]

## Classes

`class cIncomingData (aPK_obj: int | dict = None, row: dict = None, link_obj=None)`
  Expand source code

      class cIncomingData(gmBusinessDBObject.cBusinessDBObject):
              """Represents items of incoming data, say, HL7 snippets."""

              _cmd_fetch_payload = _SQL_get_incoming_data % "pk_incoming_data = %s"
              _cmds_store_payload = [
                      """UPDATE clin.incoming_data SET
                                      fk_patient_candidates = %(pk_patient_candidates)s,
                                      fk_identity = %(pk_identity)s,
                                      fk_provider_disambiguated = %(pk_provider_disambiguated)s,
                                      request_id = gm.nullify_empty_string(%(request_id)s),
                                      firstnames = gm.nullify_empty_string(%(firstnames)s),
                                      lastnames = gm.nullify_empty_string(%(lastnames)s),
                                      dob = %(dob)s,
                                      postcode = gm.nullify_empty_string(%(postcode)s),
                                      other_info = gm.nullify_empty_string(%(other_info)s),
                                      type = gm.nullify_empty_string(%(data_type)s),
                                      gender = gm.nullify_empty_string(%(gender)s),
                                      requestor = gm.nullify_empty_string(%(requestor)s),
                                      external_data_id = gm.nullify_empty_string(%(external_data_id)s),
                                      comment = gm.nullify_empty_string(%(comment)s)
                              WHERE
                                      pk = %(pk_incoming_data)s
                                              AND
                                      xmin = %(xmin_incoming_data)s
                              RETURNING
                                      xmin as xmin_incoming_data,
                                      octet_length(data) as data_size
                      """
              ]
              # view columns that can be updated:
              _updatable_fields = [
                      'pk_patient_candidates',
                      'request_id',                                           # request ID as found in <data>
                      'firstnames',
                      'lastnames',
                      'dob',
                      'postcode',
                      'other_info',                                           # other identifying info in .data
                      'data_type',
                      'gender',
                      'requestor',                                            # Requestor of data (e.g. who ordered test results) if available in source data.
                      'external_data_id',                             # ID of content of .data in external system (e.g. importer) where appropriate
                      'comment',                                                      # a free text comment on this row, eg. why is it here, error logs etc
                      'pk_identity',
                      'pk_provider_disambiguated'             # The provider the data is relevant to.
              ]
              #--------------------------------------------------------
              def format(self):
                      return '%s' % self
              #--------------------------------------------------------
              def _format_patient_identification(self):
                      tmp = '%s %s %s' % (
                              gmTools.coalesce(self._payload['lastnames'], '', 'last=%s'),
                              gmTools.coalesce(self._payload['firstnames'], '', 'first=%s'),
                              gmTools.coalesce(self._payload['gender'], '', 'gender=%s')
                      )
                      if self._payload['dob']:
                              tmp += ' dob=%s' % self._payload['dob'].strftime('%Y %b %d')
                      return tmp

              patient_identification = property(_format_patient_identification)

              #--------------------------------------------------------
              def update_data_from_file(self, fname=None, link_obj=None, verify_import:bool=False):
                      # sanity check
                      if not (os.access(fname, os.R_OK) and os.path.isfile(fname)):
                              _log.error('[%s] is not a readable file' % fname)
                              return False

                      _log.debug('updating [pk=%s] from [%s]', self.pk_obj, fname)
                      gmPG2.file2bytea (
                              query = "UPDATE clin.incoming_data SET data = %(data)s::bytea WHERE pk = %(pk)s",
                              filename = fname,
                              args = {'pk': self.pk_obj},
                              conn = link_obj
                      )
                      # must update XMIN now ...
                      self.refetch_payload(link_obj = link_obj)
                      if not verify_import:
                              return True

                      SQL = 'SELECT (md5(data) = %(local_md5)s) AS verified FROM clin.incoming_data WHERE pk = %(pk)s'
                      args = {'pk': self.pk_obj, 'local_md5': gmTools.file2md5(filename = fname)}
                      rows = gmPG2.run_ro_query(sql = SQL, args = args)
                      return rows[0]['verified']

              #--------------------------------------------------------
              def save_to_file(self, aChunkSize=0, filename=None):

                      if self._payload['data_size'] == 0:
                              return None

                      if self._payload['data_size'] is None:
                              return None

                      if filename is None:
                              filename = gmTools.get_unique_filename(prefix = 'gm-incoming_data-')

                      success = gmPG2.bytea2file (
                              data_query = {
                                      'sql': 'SELECT substring(data from %(start)s for %(size)s) FROM clin.incoming_data WHERE pk = %(pk)s',
                                      'args': {'pk': self.pk_obj}
                              },
                              filename = filename,
                              chunk_size = aChunkSize,
                              data_size = self._payload['data_size']
                      )

                      if not success:
                              return None

                      return filename

              #--------------------------------------------------------
              def lock(self, exclusive=False):
                      return gmPG2.lock_row(table = 'clin.incoming_data', pk = self.pk_obj, exclusive = exclusive)

              #--------------------------------------------------------
              def unlock(self, exclusive=False):
                      return gmPG2.unlock_row(table = 'clin.incoming_data', pk = self.pk_obj, exclusive = exclusive)

              #--------------------------------------------------------
              def set_patient(self, patient):
                      if patient is None:
                              pk_pat = None
                      elif isinstance(patient, int):
                              pk_pat = patient
                      else:
                              pk_pat = patient['pk_identity']
                      if self['pk_identity'] == pk_pat:
                              return

                      self['pk_identity'] = pk_pat

              patient = property(fset = set_patient)

  Represents items of incoming data, say, HL7 snippets.

  Call init from child classes like so:

          super().__init__(aPK_obj = aPK_obj, row = row, link_obj = link_obj)

  ## Args

  `aPK_obj`
    retrieve data from backend
  - an scalar value the ._cmd_fetch_payload WHERE condition must be a simple column: "… WHERE pk_col = %s"
  - a dictionary of values the ._cmd_fetch_payload WHERE condition must consume the dictionary and produce a unique row
  `row`
    must hold the fields
  - data: list of column values for the row selected by ._cmd_fetch_payload (as returned by cursor.fetchone() in the DB-API)
  - pk_field: the name of the primary key column OR
  - pk_obj: a dictionary suitable for being passed to cursor.execute and holding the primary key values, used for composite PKs
  - for example:

        row = {
                'data': rows[0],
                'pk_field': 'pk_XXX (the PK column name)',
                'pk_obj': {'pk_col1': pk_col1_val, 'pk_col2': pk_col2_val}
        }

        rows = gmPG2.run_ro_queries(queries = [{'sql': cmd, 'args': args}])
        objects = [ cChildClass(row = {'data': r, 'pk_field': 'the PK column name'}) for r in rows ]

  ### Ancestors

  - cBusinessDBObject

  ### Instance variables

  `prop patient`
    Expand source code

        def set_patient(self, patient):
                if patient is None:
                        pk_pat = None
                elif isinstance(patient, int):
                        pk_pat = patient
                else:
                        pk_pat = patient['pk_identity']
                if self['pk_identity'] == pk_pat:
                        return

                self['pk_identity'] = pk_pat

  `prop patient_identification`
    Expand source code

        def _format_patient_identification(self):
                tmp = '%s %s %s' % (
                        gmTools.coalesce(self._payload['lastnames'], '', 'last=%s'),
                        gmTools.coalesce(self._payload['firstnames'], '', 'first=%s'),
                        gmTools.coalesce(self._payload['gender'], '', 'gender=%s')
                )
                if self._payload['dob']:
                        tmp += ' dob=%s' % self._payload['dob'].strftime('%Y %b %d')
                return tmp

  ### Methods

  `def lock(self, exclusive=False)`
    Expand source code

        def lock(self, exclusive=False):
                return gmPG2.lock_row(table = 'clin.incoming_data', pk = self.pk_obj, exclusive = exclusive)

  `def save_to_file(self, aChunkSize=0, filename=None)`
    Expand source code

        def save_to_file(self, aChunkSize=0, filename=None):

                if self._payload['data_size'] == 0:
                        return None

                if self._payload['data_size'] is None:
                        return None

                if filename is None:
                        filename = gmTools.get_unique_filename(prefix = 'gm-incoming_data-')

                success = gmPG2.bytea2file (
                        data_query = {
                                'sql': 'SELECT substring(data from %(start)s for %(size)s) FROM clin.incoming_data WHERE pk = %(pk)s',
                                'args': {'pk': self.pk_obj}
                        },
                        filename = filename,
                        chunk_size = aChunkSize,
                        data_size = self._payload['data_size']
                )

                if not success:
                        return None

                return filename

  `def set_patient(self, patient)`
    Expand source code

        def set_patient(self, patient):
                if patient is None:
                        pk_pat = None
                elif isinstance(patient, int):
                        pk_pat = patient
                else:
                        pk_pat = patient['pk_identity']
                if self['pk_identity'] == pk_pat:
                        return

                self['pk_identity'] = pk_pat

  `def unlock(self, exclusive=False)`
    Expand source code

        def unlock(self, exclusive=False):
                return gmPG2.unlock_row(table = 'clin.incoming_data', pk = self.pk_obj, exclusive = exclusive)

  `def update_data_from_file(self, fname=None, link_obj=None, verify_import: bool = False)`
    Expand source code

        def update_data_from_file(self, fname=None, link_obj=None, verify_import:bool=False):
                # sanity check
                if not (os.access(fname, os.R_OK) and os.path.isfile(fname)):
                        _log.error('[%s] is not a readable file' % fname)
                        return False

                _log.debug('updating [pk=%s] from [%s]', self.pk_obj, fname)
                gmPG2.file2bytea (
                        query = "UPDATE clin.incoming_data SET data = %(data)s::bytea WHERE pk = %(pk)s",
                        filename = fname,
                        args = {'pk': self.pk_obj},
                        conn = link_obj
                )
                # must update XMIN now ...
                self.refetch_payload(link_obj = link_obj)
                if not verify_import:
                        return True

                SQL = 'SELECT (md5(data) = %(local_md5)s) AS verified FROM clin.incoming_data WHERE pk = %(pk)s'
                args = {'pk': self.pk_obj, 'local_md5': gmTools.file2md5(filename = fname)}
                rows = gmPG2.run_ro_query(sql = SQL, args = args)
                return rows[0]['verified']

  ### Inherited members

  - `cBusinessDBObject`:
    - `fields_as_dict`
    - `format`
    - `get_fields`
    - `get_patient`
    - `get_updatable_fields`
    - `is_modified`
    - `keys`
    - `patient_pk`
    - `refetch_payload`
    - `same_payload`
    - `save`
    - `save_payload`
    - `staff_id`