Source code for findylab.findyIoT

"""findyIoT - the HiGPS communication protocol.

The :class:`findyIoT` class below is **verbatim** from the device
firmware (``firmware_core.py`` of Logger Pro and the findyIoT module). On a device a single
instance exists as the module global ``protocol``; its methods reference
the firmware module globals (``modem``, ``db``, ``device``, ``Gimei``,
``connectionSuccess``, ``wdt``/``wdtAlert``...) exactly as shown here.

Which parameters a report carries is decided by the application: the
``datalist`` attribute is set from ``fw.report(alarm, datalist)`` and
the list itself lives in the user-editable ``last.py``.
"""

import time
import utime
import struct

import machine

from findylab.ota import (OTA_CHUNK, OTA_ZIP, OTA_META, OTA_STAGE, OTA_COMMIT,
                          OTA_APPLIED, ota_zip_ok, ota_extract, _ota_rmtree)

#: Re-entrancy guard for :meth:`findyIoT.downloadUpdate` (module global on the device).
_ota_busy = False
#: Device IMEI (set during first boot on a device).
Gimei = "0"
#: Firmware version string appended to every report.
firmwareVersion = "3_5_4"
#: Accumulated error text, sent as the ``&er=`` request field.
errors = ""
errorsSent = True
#: True after the last request reached the server.
connectionSuccess = False
registrationTime = 0
timestamp = 0
timeOffset = 0
#: Parameter IDs sent once at first boot (the ``defaults`` report).
defaults = ['421','422','524','525','553','554','555','556','523','575','576','577','578','579','580','581','587','636','637','638','639','640']


[docs] class findyIoT(object): """HiGPS protocol engine (the ``protocol`` instance on a device). Verbatim class from ``firmware_core.py``. Requests are built by the ``get_<type>`` methods and sent through the global ``modem``; server replies (``*SET``/``*GET``/...) dispatch to ``set_<type>`` handlers via :meth:`parse`. Attributes: datalist (list): Parameter IDs sent by :meth:`get_samplings`. Populated from ``fw.report(alarm, datalist)``; the list itself is owned by ``last.py``. Note: Methods reference the firmware_core module globals (``modem``, ``db``, ``Gimei``, ``connectionSuccess``, ``wdt``...), exactly as on the device. """ # Universal guard against command-dispatch recursion. An E-type command # whose handler reports back via self.main() re-enters send(); a server # that keeps answering with the same *SET,<cmd> (seen with a stuck 51/55 # during OTA testing) would otherwise nest send()->set_set()->handler-> # main()->send()... until the MicroPython stack overflows. We cap the # nesting depth and stop dispatching past it instead of crashing. Normal # command chains delivered one-per-poll by the server run in main()'s # iterative while-loop at depth 0-1, so this never limits them. _dispatch_depth = 0 _DISPATCH_MAX = 5
[docs] def __init__(self): """Initializes the protocol state (request type, IMEI, datalist).""" findyIoT.requestType = "" findyIoT.responceType = "" #higps.modem = modem #sim7000(26,23,32,21,9600) findyIoT.imei = Gimei # Which parameter IDs get_samplings() sends. fw.report() sets this # from its `datalist` argument (the list itself lives in last.py). findyIoT.datalist = []
#higps.modem.isOn()
[docs] def main(self,command,execute = True): """Sends a request and processes chained server commands. Keeps calling :meth:`send` while the server keeps returning a follow-up command, so one report can carry out a whole remote-configuration session. Args: command (str): Request type - ``'samplings'``, ``'samplingsAlarm'``, ``'defaults'``, ``'dataBat'``, ``'setting,<id>'`` or ``'command,<id>'``. execute (bool): When False, server commands are received but not executed (used for status echoes). Returns: The final :meth:`send` result (falsy when the chain ended). """ global lastReportTime while command: command = self.send(command,execute) lastReportTime = utime.time() print("Last report was on ") print(lastReportTime) return command
[docs] def send(self,commandtype,execute): """Sends a single request to the HiGPS server. Builds the body via the matching ``get_<type>`` method and performs the GET. A bare 10-digit reply is a server timestamp and synchronizes the RTC (also sets the global ``timeOffset``); a command reply dispatches to the matching ``set_<type>`` handler - at most ``_DISPATCH_MAX`` levels deep, so a server looping the same ``*SET`` cannot overflow the stack. Sets the global ``connectionSuccess``. Args: commandtype (str): Request type, optionally with a comma parameter (``'setting,421'``). execute (bool): Whether to act on the server reply. Returns: str: The follow-up request to send, or False when done. """ global lastReportTime global errors global errorsSent global connectionSuccess global registrationTime global timestamp global firmwareVersion global timeOffset connectionSuccess = False if "," in commandtype : spl = commandtype.split(",") commandtype = spl[0] parameter = spl[1] symbols = {"self": self,"parameter" : parameter} data = eval("self.get_"+commandtype+"(parameter)", symbols) else: symbols = {"self": self} data = eval("self.get_"+commandtype+"()", symbols) modem.connectHiGPS(2) response = modem.sendHiGPS('/input.php?'+data+"&er="+errors.replace(' ','%')+"&rt="+str(registrationTime)+"&t=1&fw="+firmwareVersion) #timestamp = modem.sendHiGPS('/t/') #print(timestamp) #modem.cipClose() print("Response in send") print(response) if response == False: print("retry") #modem.reset() #modem.turnOff() #modem.turnOn() if modem.isOn(): if modem.registerNetwork(): modem.connectHiGPS(2) response = modem.sendHiGPS('/input.php?'+data+"&t=1")#+"&error="+errors.replace(' ','%')) if response: response = response.decode("utf-8") connectionSuccess = True print("SENT OK") errorsSent = True errors = "" command = self.parse(response) #if len #global wdt #wdt.feed() print(command) # ok else : command = False print(command) if execute and command == False and response and len(response) == 10 : timestamp = response print(response) #decodedTs = response.decode("utf-8") if len(response) == 10: timeDevice =device.getTimestamp() ttt=time.localtime(int(response)-946684800) machine.RTC().init((ttt[0],ttt[1],ttt[2],ttt[3],ttt[4],ttt[5],0,0)) timeServer = device.getTimestamp() print("Time Device ", timeServer) timeOffset = timeServer - timeDevice print("Time Offset ",timeOffset) print(machine.RTC().datetime()) #modem.cipClose() if command and execute : if findyIoT._dispatch_depth >= findyIoT._DISPATCH_MAX: # Too many nested command dispatches - a server is looping the # same *SET. Stop here so the stack can unwind instead of # overflowing; the outer report already ran and state is saved. print("HiGPS: command nesting too deep (", findyIoT._dispatch_depth, ") - stopping to avoid stack overflow") return False findyIoT._dispatch_depth += 1 try: symbols = {"self": self,"data": response} return eval("self.set_"+command+"(data)", symbols) finally: findyIoT._dispatch_depth -= 1
#else: # return command
[docs] def parse(self,rsp) : #from machine import WDT #wdt = WDT(timeout=2000) # enable it with a timeout of 2 seconds #wdt.feed() """Classifies a server reply. Args: rsp (str): Decoded server reply. Returns: str: One of ``'set'``, ``'get'``, ``'start'``, ``'stop'``, ``'mode'``, ``'modeQ'``, ``'gprs'``, ``'eng'``, ``'wifi'``, ``'user'``, ``'phones'`` - or False when the reply carries no command. """ if "#" in rsp : if "#User=" in rsp : parsed = "user" elif "#+" in rsp : parsed ="phones" else : parsed =False elif "*" in rsp: if "*MODE-" in rsp : parsed ="mode" elif "*MODE?$" in rsp : parsed ="modeQ" elif "*GPRS$" in rsp : parsed ="gprs" elif "*GSM$" in rsp : parsed ="eng" elif "*WIFI$" in rsp : parsed ="wifi" elif "*START" in rsp : parsed ="start" elif "*STOP" in rsp : parsed ="stop" elif "*SET" in rsp: parsed = 'set' elif "*GET" in rsp: parsed = 'get' else : parsed =False else : parsed = False # print(parsed) return parsed
[docs] def set_set(self,response): """Executes a ``*SET`` server command. ``*SET,<id>$`` executes an "E"-type parameter's function; ``*SET,<id>,<value>$`` writes a "W"-type parameter and persists the database. Args: response (str): The raw ``*SET`` reply. Returns: str: A ``'setting,<id>'`` echo request, or False. """ print(response) #response = response.decode("latin-1", 'ignore') response = response.replace('$','') splitted = response.split(",") if len(splitted) == 2 : print("Execute command") print(splitted[1]) parameterData = db.getParameterByHiGPS(splitted[1]) print(parameterData) if(parameterData[0] == "E"): print(parameterData[3]) #eval("device.device.memoryTotal()") eval("self."+parameterData[3]) #return "command,"+str(splitted[1]) #execute elif len(splitted) == 3 : print("Write setting" + splitted[1] +" = "+ splitted[2]) parameterData = db.getParameterByHiGPS(splitted[1]) if(parameterData[0] == "W"): print(parameterData[0]) db.write(splitted[1],splitted[2]) db.store() return "setting,"+str(splitted[1]) else : return False
[docs] def set_get(self,response): #response = response.decode("latin-1", 'ignore') """Executes a ``*GET`` server command (read one parameter). Args: response (str): The raw ``*GET`` reply. Returns: str: A ``'setting,<id>'`` upload request, or False. """ response = response.replace('$','') splitted = response.split(",") print("Get Measurement" + splitted[1]) parameterData = db.getParameterByHiGPS(splitted[1]) if parameterData[0] == "W" or parameterData[0] == "R": value = db.read(splitted[1]) return "setting,"+str(splitted[1]) else : return False
[docs] def get_defaults(self): """Builds the first-boot report with the ``defaults`` parameter set. Returns: str: Request body (``IMEI=...&bat=...&data=...``). """ global defaults try: bat= modem.getBat() except: bat= modem.getBat() batPercent = str(bat[1]) message = "IMEI="+str(Gimei)+"&bat="+batPercent+"&data="+self.get_data(defaults) return message
[docs] def get_samplings(self): #global wakeCause #global resetCause #global causeSent """Builds the standard report body. Sends the parameters in :attr:`datalist` (set by ``fw.report``) plus battery level. Works on a copy of the list; the pulse/analog extras (437/540, 439/541, 594/520, 570/595) are appended when the enable flags 636-639 are set. Returns: str: Request body (``IMEI=...&bat=...&data=...``). """ try: bat= modem.getBat() except: bat= modem.getBat() batPercent = str(bat[1]) batVolt = str(bat[2]) # self.datalist comes from fw.report(alarm, datalist) - the user's # list in last.py. Work on a copy: appending the 636-639 extras to # the original would duplicate them on every report in this wake. datalist = list(self.datalist) if db.read('636') == b'1': datalist.append('437') datalist.append('540') if db.read('637') == b'1': datalist.append('439') datalist.append('541') if db.read('638') == b'1': datalist.append('594') datalist.append('520') if db.read('639') == b'1': datalist.append('570') datalist.append('595') message = "IMEI="+str(Gimei)+"&bat="+batPercent+"&data="+self.get_data(datalist) return message
[docs] def get_samplingsAlarm(self): """Builds an alarm report: :meth:`get_samplings` plus the ``440,1`` flag. Returns: str: Request body with the alarm marker appended. """ return self.get_samplings()+"440,1;"
[docs] def get_dataBat(self): """Builds the minimal fallback report (IMEI + battery only). Used when building the full report failed, so the server still hears from the device. Returns: str: Request body. """ global Gimei try: bat= modem.getBat() except: bat= modem.getBat() batPercent = str(bat[1]) batVolt = str(bat[2]) message = 'IMEI='+str(Gimei)+'&bat='+batPercent+'&batVolt='+batVolt return message
[docs] def get_data(self,datalist): """Serializes parameter values into the wire format. History buffers (594/595) are unpacked from packed floats; pulse ring-buffers (540/541) are serialized with their header fields (``t_``/``p_``/``i_``). Args: datalist (list): Parameter IDs to include; unknown IDs are skipped. Returns: str: ``"<id>,<value>;"`` pairs. """ import ubinascii data = "" for par in datalist: if par in db.params: print(par) if par == '594' or par =='595': raw = db.read(par) print(raw.split(b'\xff\xff')) value = "[" for val in raw.split(b'\xff\xff'): try: num = struct.unpack('f',val) print(num) value += str(num[0]) value += ":" except: print("Except ", val) if val != b'': value += ":" value = value[:-1] + "]" elif par == '540' or par == '541': raw = buffer.get(par) print("RAW BUFFER ", raw) if raw: period = raw[0] timestamp = raw[1] initialValue = raw[2] bufferData = raw[3] #raw.split(b'\xff\xff') value = "[" #raw.split(b'\xff\xff') #decodedList = [] numBytes = len(bufferData) for i in range(0, numBytes, 2): #decodedList.append(int.from_bytes(encodedBytes[i:i+2], 'big')) value += str(int.from_bytes(bufferData[i:i+2], 'big')) value += ":" value = value[:-1] + "]" value += ";t_"+par+","+str(timestamp) value += ";p_"+par+","+str(period) value += ";i_"+par+","+str(initialValue) else: value = db.read(par).decode("utf-8") if value: value=value else : value="" data += par+","+value+";" return data
[docs] def get_setting(self,id): """Builds a single-parameter upload (``&set=`` request). Args: id (str): Parameter ID; must be readable ("R" or "W"). Returns: str: Request body, or None for non-readable parameters. """ parameterData = db.getParameterByHiGPS(id) if parameterData[0] == "W" or parameterData[0] == "R": datalist = [id] message = "IMEI="+str(Gimei)+"&set="+self.get_data(datalist) print(message) return message
[docs] def get_command(self,id): """Builds a command acknowledgement request. Args: id (str): Parameter ID of an "E"-type command. Returns: str: Request body, or None for non-command parameters. """ parameterData = db.getParameterByHiGPS(id) if parameterData[0] == "E": datalist = [0] message = "IMEI="+str(Gimei)+"&set="+id print(message) return message
[docs] def downloadUpdate(self): """Downloads the OTA package named in parameter 51 (command 55). Re-entrancy wrapper around the actual download: the completion acknowledgement (``command,55``) is sent with ``execute=True`` so a queued ``*SET,52`` flows straight into :meth:`update`; a server that kept answering with ``*SET,55`` would otherwise re-enter this method until the stack overflowed. The download itself is zip-based, resumable per chunk (the partial file is pinned to the package via ``update.meta``) and CRC32-verified before parameter 53 reports *ready* (2). Progress and errors go to parameters 53/54 - see :doc:`/guides/ota-update`. Returns: bool: True when the package is downloaded and verified. """ # Re-entrancy guard. The command-55 completion ack below # (self.main("command,55")) runs with execute=True so the server's # reply is processed in-line - that lets a queued *SET,52 flow # straight into update(). But a server that keeps answering the ack # with *SET,55 would re-enter downloadUpdate over and over until the # MicroPython stack overflows (seen in the field). Allow one level # only: a nested 55 returns immediately, the outer call already # reported 53=2, and the next report cycle can still pick up 52. global _ota_busy if _ota_busy: print("OTA: 55 re-entry blocked (already in download/ack)") return False _ota_busy = True try: return self._downloadUpdate() finally: _ota_busy = False
def _downloadUpdate(self): """The download proper - see :meth:`downloadUpdate`. Never touches live files: it only writes ``OTA_ZIP`` / ``OTA_META``. Result codes written to 54: 4 connection lost (partial kept), 5 integrity failure (discarded), 6 unsupported package, 7 invalid size. """ # Command 55 - download the firmware package named in '51' into # OTA_ZIP. Resumable (the byte offset is the file size on flash, # pinned to the package via OTA_META) and integrity-checked (every # zip entry's CRC32) before it is ever marked ready. Never touches # live files - applying the package is command 52 (update()). import os import json name = db.read('51') if name == False or name == b'' or name == b'0': print("OTA: no package name in '51'") return False name = name.decode('utf-8') # Zip-only going forward, but the server names packages without a # ".zip" suffix (e.g. "fw2608251537"), so we don't gate on the # extension - the whole-archive CRC32 check below is what proves the # payload really is an intact zip before anything is applied. wdt = machine.Timer(1) wdt.init(period=90000, mode=machine.Timer.ONE_SHOT, callback=wdtAlert) print("OTA: starting download of", name) db.write('53', '1') db.write('54', '0') db.store() self.main("setting,53", False) # Total package size from the repo. modem.connectHiGPS(2) sizeResp = modem.sendHiGPS('/repo/' + name) modem.cipClose() if hasattr(sizeResp, 'decode'): sizeResp = sizeResp.decode('utf-8') try: total = int(str(sizeResp).strip()) except (ValueError, TypeError): total = 0 if total <= 0: print("OTA: bad size reply", sizeResp) db.write('53', '0') db.write('54', '7') # invalid size / URI db.store() self.main("setting,54", False) return False print("OTA: package size", total) # Resume only if the on-flash partial belongs to THIS package and # was written with THIS chunk size; otherwise start clean. meta_ok = False try: m = json.loads(open(OTA_META).read()) meta_ok = (m.get('name') == name and m.get('chunk') == OTA_CHUNK and m.get('size') == total) except (OSError, ValueError): meta_ok = False resume = 0 complete = False if meta_ok: try: have = os.stat(OTA_ZIP)[6] except OSError: have = 0 if have == total: # Already fully downloaded (e.g. power loss after the last # chunk but before verify) - skip straight to verification. complete = True elif have < total and have % OTA_CHUNK == 0: # Clean chunk boundary - resume from here. resume = have // OTA_CHUNK else: # Torn tail or oversized - can't trust it, start clean. meta_ok = False if not meta_ok: _ota_rmtree(OTA_ZIP) _ota_rmtree(OTA_META) try: open(OTA_META, 'w').write(json.dumps( {'name': name, 'size': total, 'chunk': OTA_CHUNK})) except OSError: pass open(OTA_ZIP, 'wb').close() resume = 0 complete = False if not complete: print("OTA: resuming from chunk", resume, "of", (total + OTA_CHUNK - 1) // OTA_CHUNK) f = open(OTA_ZIP, 'ab') n = resume retries = 3 try: while n * OTA_CHUNK < total: wdt.init(period=90000, mode=machine.Timer.ONE_SHOT, callback=wdtAlert) expected = total - n * OTA_CHUNK if expected > OTA_CHUNK: expected = OTA_CHUNK chunk = b'' for attempt in range(retries): modem.connectHiGPS(2) chunk = modem.downloadChunk( '/repo/%s/%d/%d' % (name, OTA_CHUNK, n)+'/'+Gimei, expected) modem.cipClose() if len(chunk) == expected: break print("OTA: chunk", n, "got", len(chunk), "of", expected, "- retry", attempt + 1) if not modem.isOn(): modem.turnOn() modem.registerNetwork() if len(chunk) != expected: # Give up for now but KEEP the partial file: the next # 55 (or a power-up) resumes here instead of restarting. print("OTA: chunk", n, "failed - will resume later") db.write('53', '0') db.write('54', '4') # connection lost db.store() self.main("setting,54", False) return False f.write(chunk) f.flush() n += 1 finally: f.close() # Whole-archive integrity (per-entry CRC32) before marking ready. ok = ota_zip_ok(OTA_ZIP) if ok is None: print("OTA: archive needs deflate support this build lacks") db.write('53', '0') db.write('54', '6') # unsupported package type db.store() self.main("setting,54", False) return False if not ok: print("OTA: integrity check failed - discarding download") _ota_rmtree(OTA_ZIP) _ota_rmtree(OTA_META) db.write('53', '0') db.write('54', '5') # integrity check failure db.store() self.main("setting,54", False) return False db.write('53', '2') # downloaded / ready db.write('54', '0') db.store() self.main("setting,53", False) self.main("setting,54", False) print("OTA: download complete and verified - ready to apply (52)") # Acknowledge that command 55 finished. execute=True so the server's # reply is processed in-line: once it sees 55 done it sends *SET,52, # which is dispatched straight into update() here. (If the server is # misconfigured to keep replying *SET,55, downloadUpdate re-enters on # the already-complete package - a fast re-verify - and the 90 s WDT # is the backstop; the normal path is 55 -> 52 as seen in the field.) self.main("command,55") return True
[docs] def update(self): """Applies the downloaded package (command 52). Re-verifies the archive, extracts it into the ``OTA_STAGE`` directory (each file CRC32-checked as written) and creates the ``OTA_COMMIT`` marker; ``boot.py`` then copies the staged tree onto the live filesystem on the next boot. Parameter 53 is 3 while updating; a failure leaves 53 at 2 (downloaded) with 54 = 8 so the server can retry. Resets the device on success. Returns: bool: False when there is no valid package or staging failed. """ # Command 52 - apply the downloaded package. Extracts into a staging # dir and drops a single commit marker; the actual swap onto the live # tree happens idempotently at boot (see boot.py), so a power loss # mid-apply can never leave a half-updated file set live. import os import gc gc.collect() print("OTA: applying update") if ota_zip_ok(OTA_ZIP) is not True: print("OTA: no valid package to apply") db.write('53', '0') db.write('54', '5') db.store() self.main("setting,53", False) self.main("setting,54", False) return False wdt = machine.Timer(1) wdt.init(period=120000, mode=machine.Timer.ONE_SHOT, callback=wdtAlert) db.write('53', '3') # updating db.write('54', '0') db.store() self.main("setting,53", False) _ota_rmtree(OTA_STAGE) try: os.mkdir(OTA_STAGE) except OSError: pass if not ota_extract(OTA_ZIP, OTA_STAGE): print("OTA: extraction failed - keeping package for retry") _ota_rmtree(OTA_STAGE) db.write('53', '2') # back to 'downloaded' - 52 can retry db.write('54', '8') # update failed db.store() self.main("setting,54", False) return False # Commit point: a single small-file create. boot.py copies OTA_STAGE # onto the live tree on the next boot and clears these markers. try: fo = open(OTA_COMMIT, 'w') fo.write(db.read('51').decode('utf-8')) fo.close() except OSError as e: print("OTA: could not write commit marker", e) _ota_rmtree(OTA_STAGE) db.write('53', '2') db.write('54', '8') db.store() self.main("setting,54", False) return False # Free the archive - OTA_STAGE is authoritative from here on. _ota_rmtree(OTA_ZIP) _ota_rmtree(OTA_META) db.store() # Acknowledge command 52 so the server stops re-queuing the apply. self.main("command,52", False) print("OTA: staged and committed - rebooting to swap files") time.sleep(2) machine.reset() return True
[docs] def resetCount1(self): """Resets the pulse counters (command 429). Re-initializes the ULP program, resets the modem, re-registers and acknowledges the command to the server. Returns: bool: True. """ timestamp = device.getTimestamp() #modem.turnOff() counter.initCounter() modem.reset() #modem.turnOn() modem.registerNetwork() #db.write('443',timestamp) self.main("command,429") #device.resetValGyro(self.db) return True
[docs] def updateTime(self): """Refreshes the device timestamp and acknowledges with command 8.""" device.getTimestamp() self.main("command,8")
[docs] def serialNumber(): """Uploads the chip serial number as a setting. Note: Inherited quirk: defined without ``self`` and references ``self`` in the body - calling it raises. Kept verbatim. """ self.main("setting,"+str(device.serialNumber())) return True
def restart(self): """Acknowledges the restart command (221) and resets the device. Note: Shadowed by the second ``restart`` definition below - on the device the later definition wins. Kept verbatim. """ self.main("command,221",False) machine.reset() return True
[docs] def factoryReset(self): """Acknowledges command 222, wipes the database and resets.""" self.main("command,222",False) device.factoryReset() return True
[docs] def restart(self): """Restarts the modem radio (``AT+CFUN=0`` / ``AT+CFUN=1``). Note: Inherited quirk: this second definition shadows the command-221 handler above and references ``self.us`` which findyIoT does not have. Kept verbatim. """ self.us("AT+CFUN=0",2) time.sleep(1) self.us("AT+CFUN=1",2)