update

tool/journalUtils.jq

@@ -10,7 +10,7 @@ def formatExperienceTitle:
 
 def formatIngestionDose($customUnits):
   . as $ingestion |
-  . | journalLib::calculateIngestionDose($customUnits) as $dose |
+  . | journalLib::ingestionDose($customUnits) as $dose |
   . | journalLib::ingestionUnit($customUnits) as $unit |
   $customUnits | map(select(.id == $ingestion.customUnitId))[0] as $customUnit |
   if $ingestion.dose == null then

tool/lib/journalLib.jq

@@ -1,17 +1,50 @@
 import "utilsLib" as utilsLib;
+import "journalTypes" as journalTypes;
 
-def calculateIngestionDose($customUnits):
+def ingestionDose($customUnits):
   . as $ingestion |
+  journalTypes::ensureIngestion |
+
   if .customUnitId != null then
     ($customUnits | map(select(.id == $ingestion.customUnitId))[0]) as $customUnit |
     .dose * $customUnit.dose | . as $dose | 
-    $dose * 100 | round / 100
+    $dose
   else 
     .dose
   end;
 
+def addStandardDeviations($expectationX; $standardDeviationX; $expectationY; $standardDeviationY):
+  (pow($standardDeviationX; 2) + pow($expectationX; 2)) as $sumX |
+  (pow($standardDeviationY; 2) + pow($expectationY; 2)) as $sumY |
+
+  (pow($expectationX; 2) * pow($expectationY; 2)) as $expectations |
+  ($sumX * $sumY - $expectations) as $productVariance |
+  if $productVariance > 0.0000001 then
+    $productVariance | sqrt
+  else 
+    null
+  end;
+
+def ingestionStandardDeviation($customUnits):
+  . as $ingestion |
+  journalTypes::ensureIngestion |
+
+  if .customUnitId != null then
+    ($customUnits | map(select(.id == $ingestion.customUnitId))[0]) as $customUnit |
+
+    ($ingestion | ingestionDose($customUnits) // 0) as $expectationX |
+    ($ingestion.estimatedDoseStandardDeviation // 0) as $standardDeviationX |
+    ($customUnit.dose // 0) as $expectationY |
+    ($customUnit.estimatedDoseStandardDeviation // 0) as $standardDeviationY |
+
+    addStandardDeviations($expectationX; $standardDeviationX; $expectationY; $standardDeviationY)
+  else 
+    .standardDeviation
+  end;
+
 def ingestionUnit($customUnits):
   . as $ingestion |
+  journalTypes::ensureIngestion |
   if .customUnitId != null then
     ($customUnits | map(select(.id == $ingestion.customUnitId))[0]) as $customUnit |
     $customUnit.originalUnit
@@ -21,10 +54,13 @@ def ingestionUnit($customUnits):
 
 def ingestionConsumerName:
   . as $ingestion |
+  journalTypes::ensureIngestion |
   $ingestion.consumerName // "default";
 
 def filterIngestions($substanceFilter; $consumerFilter):
   . as $ingestions |
+  journalTypes::ensureIngestions |
+
   if (($substanceFilter // [] | length) > 0) then 
     [
       $ingestions[] as $ingestion |
@@ -39,14 +75,17 @@ def filterIngestions($substanceFilter; $consumerFilter):
 
 def ingestionsSubstanceNames:
   . as $ingestions |
+  journalTypes::ensureIngestions |
   [$ingestions[].substanceName] | utilsLib::orderedUnique;
 
 def ingestionsConsumerNames:
   . as $ingestions |
+  journalTypes::ensureIngestions |
   [$ingestions[] | ingestionConsumerName] | utilsLib::orderedUnique;
 
 def ingestionsByConsumer:
   . as $ingestions |
+  journalTypes::ensureIngestions |
   ($ingestions | ingestionsConsumerNames) as $consumerNames |
   [
     $consumerNames[] as $consumerName |
@@ -58,35 +97,96 @@ def ingestionsByConsumer:
       }
   ] | from_entries;
 
+def defaultIngestionInfo:
+  {
+    unit: "",
+    dose: null,
+    isUnknown: false,
+    isEstimate: false,
+    # TODO
+    standardDeviation: null
+  };
+
+def ingestionInfo($customUnits):
+  . as $ingestion |
+  .substanceName as $name |
+  .administrationRoute as $administrationRoute |
+  . | ingestionStandardDeviation($customUnits) as $standardDeviation |
+  . | ingestionDose($customUnits) as $dose |
+  . | ingestionUnit($customUnits) as $unit |
+  . | ingestionConsumerName as $consumerName |
+
+  defaultIngestionInfo as $ingestionInfo |
+  $ingestionInfo |
+
+  if ($dose == null) then .isUnknown |= true end |
+
+  if ($standardDeviation != null) then
+    .isEstimate |= true
+  end |
+
+  .unit |= $unit |
+  .dose += $dose;
+
+def addIngestionInfo($rhs):
+  . as $lhs | 
+
+  if (.unit != "" and $rhs.unit != "") then
+    utilsLib::assert(.unit == $rhs.unit; "units mismatch, todo add reasonable conversions")
+  else 
+    .unit |= $rhs.unit
+  end |
+
+
+  if ($rhs.dose == null) then .isUnknown |= true end |
+
+  if ($rhs.standardDeviation != null) then
+    .isEstimate |= true 
+  end |
+
+  .dose += $rhs.dose;
+
+def addIngestionInfos:
+  . as $ingestionInfos |
+  reduce $ingestionInfos[] as $ingestionInfo (defaultIngestionInfo;
+    . | addIngestionInfo($ingestionInfo)
+  );
+
 def experienceStats($customUnits):
   . as $experience |
+  journalTypes::ensureExperience |
   $experience.ingestions as $ingestions |
   (reduce $ingestions[] as $ingestion ({}; . as $stats |
     $ingestion |
     .substanceName as $name |
     .administrationRoute as $administrationRoute |
-    . | calculateIngestionDose($customUnits) as $dose |
-    . | ingestionUnit($customUnits) as $unit |
     . | ingestionConsumerName as $consumerName |
 
-    $stats | .[$consumerName].[$name].[$administrationRoute] |= 
-      ($stats.[$consumerName].[$name].[$administrationRoute] // {
-        unit: "",
-        # null because null+null = null for ingestions with unknown dose which .dose is null
-        dose: null
-      }) |
-    .[$consumerName].[$name].[$administrationRoute].unit |= $unit |
-    .[$consumerName].[$name].[$administrationRoute].dose += $dose
+    $stats |
+      .[$consumerName].[$ingestion.substanceName].[$administrationRoute] as $ingestionStats |
+
+    if $ingestionStats == null then
+      ($ingestion | ingestionInfo($customUnits))
+    else
+      $ingestionStats | addIngestionInfo($ingestion | ingestionInfo($customUnits))
+    end | . as $ingestionStats |
+
+    $stats |
+      .[$consumerName].[$ingestion.substanceName].[$administrationRoute] |= $ingestionStats
   ));
 
 def statsCalculateCombinedDose($substanceName; $consumerName):
   . as $stats |
-  (.[$consumerName].[$substanceName] | [to_entries[] | .value.dose] | add) as $combinedDose | 
-  (.[$consumerName].[$substanceName] | to_entries[0] | .value.unit) as $combinedDoseUnit | 
-  {dose: $combinedDose, unit: $combinedDoseUnit};
+  utilsLib::assert($stats | has($consumerName); "consumer not found in stats") |
+  utilsLib::assert($stats[$consumerName] | has($substanceName); 
+    "substance for consumer not found in stats"
+  ) |
+
+  .[$consumerName].[$substanceName] | values | addIngestionInfos;
 
 def experiencesWithExtraData($customUnits):
   . as $experiences |
+  journalTypes::ensureExperiences |
   (reduce $experiences[] as $experience ([]; 
     ($experience | experienceStats($customUnits)) as $stats |
     . += [{
@@ -97,6 +197,7 @@ def experiencesWithExtraData($customUnits):
 
 def filterSortExperiences($customUnits; $substanceFilter; $consumerFilter; $sortMethod; $sortOptions):
   . as $experiences |
+  journalTypes::ensureExperiences |
   ($consumerFilter // ["default"]) as $consumerFilter |
   $sortOptions |
     # If filtering results by substances but no sortOptions.substanceName is defined, use the first one as a default
@@ -131,7 +232,7 @@ def filterSortExperiences($customUnits; $substanceFilter; $consumerFilter; $sort
               any(index($ingestion.substanceName))
             )
           ) then $experienceData else null end
-      ] | objects;
+      ] | map(select(. != null));
 
     def filterByConsumerFilter:
       . as $experiencesData |

tool/lib/journalLibTests.jq

@@ -0,0 +1,24 @@
+include "dropins";
+
+import "testLib" as testLib;
+import "journalLib" as journalLib;
+
+import "testdata/tests_export" as $exportDataArray;
+
+def journalLibTests:
+  testLib::expectPassed(testLib::runTest(
+    "invalid input to experienceByTitle";
+    (
+      null | journalLib::experienceByTitle("Test")
+    );
+    . == "experienceByTitle takes a array of experiences as input";
+    true
+  )) |
+  testLib::expectPassed(testLib::runTest(
+    "experience not found";
+    (
+      $exportDataArray[0].experiences | journalLib::experienceByTitle("Test")
+    );
+    . == null;
+    false
+  ));

tool/lib/journalTypes.jq

@@ -69,6 +69,12 @@ def ensureIngestion:
     $ingestion
   end;
 
+def ensureIngestions:
+  . as $ingestions |
+  if typeLib::typecheckingEnabled then
+    (reduce $ingestions[] as $ingestion (null; $ingestion | ensureIngestion))
+  end;
+
 def ensureExperience:
   . as $experience |
   if typeLib::typecheckingEnabled then
@@ -88,11 +94,17 @@ def ensureExperience:
 
     $experience | typeLib::ensureKey("experience"; "ingestions") | 
     $experience.ingestions | typeLib::ensureWrapError("experience:ingestions"; typeLib::ensureArray) |
-    (reduce $experience.ingestions[] as $ingestion (null; $ingestion | ensureIngestion)) |
+    $experience.ingestions | ensureIngestions |
 
     $experience
   end;
 
+def ensureExperiences:
+  . as $experiences |
+  if typeLib::typecheckingEnabled then
+    (reduce $experiences[] as $experience (null; $experience | ensureExperience))
+  end;
+
 def ensureExportData:
   . as $exportData |
   if typeLib::typecheckingEnabled then
@@ -101,5 +113,5 @@ def ensureExportData:
     typeLib::ensureKey("exportData"; "customUnits") |
     typeLib::ensureKey("exportData"; "experiences") |
     typeLib::ensureKey("exportData"; "substanceCompanions") |
-    (reduce .experiences[] as $experience (null; $experience | ensureExperience))
+    .experiences | ensureExperiences
   end;

tool/main.jq

@@ -44,15 +44,33 @@ def printExperienceStats($stats; $substanceFilter; $consumerFilter; $withTitle):
 
       ($experienceStatsText | . += "Substance: \($substanceName)\n") as $experienceStatsText |
 
-      reduce ($stats.[$consumerName].[$substanceName] | to_entries)[] as $substanceStats ($experienceStatsText;
+      reduce ($stats.[$consumerName].[$substanceName] | to_entries)[] as $ingestionMethodInfo ($experienceStatsText;
         . as $experienceStatsText |
 
-        $substanceStats |
+        $ingestionMethodInfo |
         .key as $ingestionMethod |
-        (.value.dose // "Unknown") as $dose |
-        .value.unit as $unit |
+        .value as $ingestionInfo |
+
+        def formatIngestionInfo:
+          if $ingestionInfo.dose == null then
+            "Unknown \($ingestionInfo.unit)"
+          else
+            if $ingestionInfo.isEstimate then "~" else "" end +
+            "\($ingestionInfo.dose * 100 | round / 100)" +
+            if $ingestionInfo.standardDeviation != null then
+              "±\($ingestionInfo.standardDeviation)"
+            else "" end +
+
+            if $ingestionInfo.isUnknown then "+ Unknown" else "" end +
+            " \($ingestionInfo.unit)"
+
+
+          end;
+
+        $experienceStatsText | 
+          . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($ingestionInfo | formatIngestionInfo)\n" |
+          . as $experienceStatsText | 
           
-        ($experienceStatsText | . += "Dose (\($ingestionMethod | stringLib::titleCase)): \($dose) \($unit)\n") as $experienceStatsText | 
         $experienceStatsText
       ) | . as $experienceStatsText |
 

tool/tests.jq

@@ -2,32 +2,10 @@ include "dropins";
 
 import "lib/testLib" as testLib;
 import "lib/typeLib" as typeLib;
-import "lib/journalLib" as journalLib;
-
-import "journalUtils" as journalUtils;
-
-import "testdata/tests_export" as $exportDataArray;
-
-def journalUtilsTests:
-  testLib::expectPassed(testLib::runTest(
-    "invalid input to experienceByTitle";
-    (
-      null | journalLib::experienceByTitle("Test")
-    );
-    . == "experienceByTitle takes a array of experiences as input";
-    true
-  )) |
-  testLib::expectPassed(testLib::runTest(
-    "experience not found";
-    (
-      $exportDataArray[0].experiences | journalLib::experienceByTitle("Test")
-    );
-    . == null;
-    false
-  ));
+import "lib/journalLibTests" as journalLibTests;
 
 def testsMain:
   testLib::testTests |
   typeLib::typeLibTests |
-  journalUtilsTests | 
+  journalLibTests::journalLibTests | 
   "Tests Passed\n" | halt_error(0);