merge master

all-of-us · YintongMa · Aug 25, 2021 · Jul 28, 2021 · Jul 28, 2021 · Jul 28, 2021
commit 47afd218acf2e566011d0569ede87f976d7b1423
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -32,9 +32,7 @@ anchors:
     TERM: dumb
 
   env-db: &env-db
-    MYSQL_ROOT_PASSWORD: ubuntu
-    MYSQL_USER: ubuntu
-    MYSQL_PASSWORD: ubuntu
+    MYSQL_ROOT_PASSWORD: root-notasecret
 
   # Job runs for master branch only
   filter-master-branch: &filter-master-branch
@@ -207,15 +205,10 @@ commands:
           # Watch the port for 2m for MySQL start up
           name: Await MySQL startup
           command: dockerize -wait tcp://127.0.0.1:3306 -timeout 2m
-      - run:
-          name: Run Local Migrations
-          working_directory: ~/workbench/api
-          command: ./project.rb run-local-migrations
       - run:
           name: Launch local API server
           working_directory: ~/workbench/api
-          # tail -f is important here, it keeps this process running indefinitely
-          command: ./project.rb start-local-api && tail -f build/dev-appserver-out/dev_appserver.out
+          command: ./project.rb dev-up --nostart-db
           background: true
       - run:
           name: Wait for local API server to start
@@ -400,7 +393,7 @@ jobs:
             - launch-local-api
             - run:
                 working_directory: ~/workbench/api
-                command: ./project.rb run-local-api-tests && ./project.rb stop-local-api
+                command: ./project.rb run-local-api-tests
 
   api-deps-check:
     executor: workbench-executor
@@ -410,7 +403,7 @@ jobs:
             - run:
                 name: Scan dependencies for vulnerabilities
                 working_directory: ~/workbench/api
-                command: ./project.rb gradle dependencyCheckAnalyze --info
+                command: ./gradlew dependencyCheckAnalyze --info
 
   api-integration-test:
     executor: workbench-executor
@@ -449,10 +442,6 @@ jobs:
           name: Yarn codegen
           working_directory: ~/workbench/ui
           command: yarn codegen
-      - run:
-          name: Run Angular tests
-          working_directory: ~/workbench/ui
-          command: yarn test --no-watch --no-progress --browsers=ChromeHeadless
       - run:
           name: Run React jest tests
           working_directory: ~/workbench/ui
@@ -552,7 +541,7 @@ jobs:
           at: .
       - steps: << parameters.optional_steps >>
       - run:
-          name: "Export e2e test user to environment variable"
+          name: "Export e2e test users to environment variables"
           command: bash .circleci/export-e2e-test-user-vars.sh << parameters.env_name >>
       - when:
           condition:
@@ -624,6 +613,17 @@ jobs:
           paths:
             - .
 
+  puppeteer-access-test-user-setup:
+    executor: workbench-executor
+    steps:
+      - checkout-code
+      - attach_workspace:
+          at: .
+      - gcloud-auth-login
+      - run:
+          working_directory: ~/workbench/api
+          name: Prep the e2e access test user (in Test env) by setting access module timestamps
+          command: ./project.rb set-access-module-timestamps --user ${PUPPETEER_ACCESS_TEST}
 
 workflows:
   build-test-deploy:
@@ -727,12 +727,14 @@ workflows:
     jobs:
       - api-bigquery-test
       - puppeteer-env-setup
+      - puppeteer-access-test-user-setup
       - puppeteer-test:
           parallel_num: 1
           env_name: "test"
           test_mode: "nightly-integration"
           requires:
             - puppeteer-env-setup
+            - puppeteer-access-test-user-setup
 
 experimental:
   notify:

diff --git a/.circleci/export-e2e-test-user-vars.sh b/.circleci/export-e2e-test-user-vars.sh
@@ -7,30 +7,34 @@ for arg in "$@"
 do
   case "$arg" in
     "perf")
-       echo "export READER_USER=$PUPPETEER_READER_PERF" >> $BASH_ENV
-       echo "export WRITER_USER=$PUPPETEER_WRITER_PERF" >> $BASH_ENV
-       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_PERF" >> $BASH_ENV
-       echo "export USER_NAME=$PUPPETEER_USER_PERF" >> $BASH_ENV
+       echo "export READER_USER=$PUPPETEER_READER_PERF" >> "$BASH_ENV"
+       echo "export WRITER_USER=$PUPPETEER_WRITER_PERF" >> "$BASH_ENV"
+       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_PERF" >> "$BASH_ENV"
+       echo "export USER_NAME=$PUPPETEER_USER_PERF" >> "$BASH_ENV"
+       echo "export ACCESS_TEST_USER=$PUPPETEER_ACCESS_PERF" >> "$BASH_ENV"
        ;;
     "staging")
-       echo "export READER_USER=$PUPPETEER_READER_STAGING" >> $BASH_ENV
-       echo "export WRITER_USER=$PUPPETEER_WRITER_STAGING" >> $BASH_ENV
-       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_STAGING" >> $BASH_ENV
-       echo "export USER_NAME=$PUPPETEER_USER_STAGING" >> $BASH_ENV
+       echo "export READER_USER=$PUPPETEER_READER_STAGING" >> "$BASH_ENV"
+       echo "export WRITER_USER=$PUPPETEER_WRITER_STAGING" >> "$BASH_ENV"
+       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_STAGING" >> "$BASH_ENV"
+       echo "export USER_NAME=$PUPPETEER_USER_STAGING" >> "$BASH_ENV"
+       echo "export ACCESS_TEST_USER=$PUPPETEER_ACCESS_STAGING" >> "$BASH_ENV"
        ;;
     "test")
-       echo "export READER_USER=$PUPPETEER_READER_TEST" >> $BASH_ENV
-       echo "export WRITER_USER=$PUPPETEER_WRITER_TEST" >> $BASH_ENV
-       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_TEST" >> $BASH_ENV
-       echo "export USER_NAME=$PUPPETEER_USER_TEST" >> $BASH_ENV
+       echo "export READER_USER=$PUPPETEER_READER_TEST" >> "$BASH_ENV"
+       echo "export WRITER_USER=$PUPPETEER_WRITER_TEST" >> "$BASH_ENV"
+       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_TEST" >> "$BASH_ENV"
+       echo "export USER_NAME=$PUPPETEER_USER_TEST" >> "$BASH_ENV"
+       echo "export ACCESS_TEST_USER=$PUPPETEER_ACCESS_TEST" >> "$BASH_ENV"
        ;;
     "local")
-       echo "export READER_USER=$PUPPETEER_READER_LOCAL" >> $BASH_ENV
-       echo "export WRITER_USER=$PUPPETEER_WRITER_LOCAL" >> $BASH_ENV
-       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_LOCAL" >> $BASH_ENV
-       echo "export USER_NAME=$PUPPETEER_USER_LOCAL" >> $BASH_ENV
+       echo "export READER_USER=$PUPPETEER_READER_LOCAL" >> "$BASH_ENV"
+       echo "export WRITER_USER=$PUPPETEER_WRITER_LOCAL" >> "$BASH_ENV"
+       echo "export COLLABORATOR_USER=$PUPPETEER_COLLABORATOR_LOCAL" >> "$BASH_ENV"
+       echo "export USER_NAME=$PUPPETEER_USER_LOCAL" >> "$BASH_ENV"
+       echo "export ACCESS_TEST_USER=$PUPPETEER_ACCESS_LOCAL" >> "$BASH_ENV"
        ;;
    esac
-   echo "export PASSWORD=$PUPPETEER_USER_PASSWORD" >> $BASH_ENV
+   echo "export PASSWORD=$PUPPETEER_USER_PASSWORD" >> "$BASH_ENV"
    source $BASH_ENV
 done
diff --git a/.circleci/workflow-queue-v1.sh b/.circleci/workflow-queue-v1.sh
@@ -54,22 +54,29 @@ circle_get() {
     "${url}"
 }
 
+# Function returns current pipeline's start_time. It is used for comparison of start_time values.
+fetch_current_pipeline_start_time() {
+  printf '%s\n' "Fetch current pipeline start time."
+  local get_path="project/${project_slug}/tree/${branch}?filter=running&shallow=true"
+  local curl_result=$(circle_get "${get_path}")
+  __=$(echo "${curl_result}" | jq -r ".[] | select(.build_num==$CIRCLE_BUILD_NUM) | .start_time")
+}
+
+# Function takes start_time parameter.
 # Fetch list of builds on master branch that are running, pending or queued.
-fetch_pipelines() {
-  printf '%s\n' "Fetch previously submitted builds on \"${branch}\" branch that are running, pending or queued:"
+fetch_older_pipelines() {
+  printf '%s\n' "Fetch pipelines (older than ${1}) on \"${branch}\" branch that are running, pending or queued."
   local get_path="project/${project_slug}/tree/${branch}?filter=running&shallow=true"
-  local get_result=$(circle_get "${get_path}")
-  if [[ ! "${get_result}" ]]; then
-    printf "curl failed."
+  local curl_result=$(circle_get "${get_path}")
+  if [[ ! "${curl_result}" ]]; then
+    printf "Fetch older pipelines failed."
     exit 1
   fi
-  jq_filter="select(.branch==\"${branch}\""
-  jq_filter+=" and .build_num < $CIRCLE_BUILD_NUM"
-  jq_filter+=" and (.status | test(\"running|pending|queued\")))"
-  jq_filter+=" | select(.workflows.workflow_name==\"${workflow_name}\""
-  jq_filter+=" and .workflows.workflow_id!=\"${CIRCLE_WORKFLOW_ID}\")"
-
-  __=$(echo "${get_result}" | jq -r ".[] | ${jq_filter} | .build_num | @sh")
+  jq_filter=".branch==\"${branch}\" and (.status | test(\"running|pending|queued\")) "
+  jq_filter+="and .workflows.workflow_name==\"${workflow_name}\" and .workflows.workflow_id!=\"${CIRCLE_WORKFLOW_ID}\""
+  jq_object="{ workflow_name: .workflows.workflow_name, workflow_id: .workflows.workflow_id, "
+  jq_object+="job_name: .workflows.job_name, build_num, start_time, status }"
+  __=$(echo "${curl_result}" | jq -r ".[] | select(${jq_filter}) | select(.start_time < \"${1}\") | ${jq_object}")
 }
 
 
@@ -85,29 +92,38 @@ check_circleci_workflow_id
 printf "%s\n" "Current pipeline CIRCLE_BUILD_NUM: ${CIRCLE_BUILD_NUM}"
 printf "%s\n\n" "Current pipeline CIRCLE_WORKFLOW_ID: ${CIRCLE_WORKFLOW_ID}"
 
+fetch_current_pipeline_start_time
+current_pipeline_start_time=$__
+printf "%s\n\n" "Current pipeline start_at time: ${current_pipeline_start_time}"
+if [[ ! "${current_pipeline_start_time}" ]]; then
+  printf "Value of current_pipeline_start_time is not valid."
+  exit 1
+fi
+
 # Wait as long as "pipelines" variable is not empty until max time has reached.
 is_running=true
 waited_time=0
 wait="30s"
 
 while [[ "${is_running}" == "true" ]]; do
   printf "\n***\n"
-  fetch_pipelines
+  fetch_older_pipelines "${current_pipeline_start_time}"
   pipelines=$__
 
   if [[ $pipelines ]]; then
-    printf "%s\n" "Waiting for previous builds to finish. sleep ${wait} seconds. Please wait..."
+    printf "\n%s\n" "${pipelines}"
+    printf "%s\n" "Waiting for previously submitted pipelines to finish. sleep ${wait} seconds. Please wait..."
     sleep $sleep_time
     waited_time=$((sleep_time + waited_time))
   else
-    printf "\n%s\n" "all previous builds have finished."
+    printf "\n%s\n" "All previously submitted pipelines have finished."
     is_running=false
   fi
 
   printf "%s\n" "Has been waiting for ${waited_time} seconds."
   if [ $waited_time -gt $max_time ]; then
-    printf "\n\n%s\n\n" "***** Max wait time (${max_time} seconds) exceeded. Stop waiting for running builds. *****"
+    printf "\n\n%s\n\n" "***** Max wait time (${max_time} seconds) exceeded. Stop waiting for running builds to complete."
     is_running=false
   fi
 done
-printf "\n%s\n" "Finished."
+printf "\n%s\n" "Finished waiting."
diff --git a/.gitignore b/.gitignore
@@ -48,6 +48,7 @@ tools/*.jar
 tools/cloud_sql_proxy
 .DS_Store
 client/py/.pydevproject
+gradle.properties
 **/.idea
 **/*.iml
 **/*.ipr

diff --git a/api/build.gradle b/api/build.gradle
@@ -1,6 +1,10 @@
 import org.pmiops.workbench.tooling.GenerateAPIListingTask
 import org.pmiops.workbench.tooling.IncrementalHotSwapTask
 
+if(JavaVersion.current() != JavaVersion.VERSION_1_8){
+  throw new GradleException("This build must be run with Java 8. See developer-system-initialization.md")
+}
+
 // Runs before all tasks. Sets up properties and dependencies for the build
 // process itself.
 buildscript {
@@ -677,16 +681,6 @@ task fetchWorkspaceDetails(type: JavaExec) {
   }
 }
 
-// See project.rb command: export-workspace-data
-task exportWorkspaceData(type: JavaExec) {
-  classpath = sourceSets.__tools__.runtimeClasspath
-  main = "org.pmiops.workbench.tools.ExportWorkspaceData"
-  systemProperties = commandLineSpringProperties
-  if (project.hasProperty("appArgs")) {
-    args Eval.me(appArgs)
-  }
-}
-
 // See project.rb command: generate-impersonated-user-tokens
 task generateImpersonatedUserTokens(type: JavaExec) {
   classpath = sourceSets.__tools__.runtimeClasspath
@@ -813,6 +807,16 @@ task fixDesynchronizedBillingProjectOwners(type: JavaExec) {
   }
 }
 
+// See project.rb command: set-access-module-timestamps
+task setAccessModuleTimestamps(type: JavaExec) {
+  classpath = sourceSets.__tools__.runtimeClasspath
+  main = "org.pmiops.workbench.tools.SetAccessModuleTimestamps"
+  systemProperties = commandLineSpringProperties
+  if (project.hasProperty("appArgs")) {
+    args Eval.me(appArgs)
+  }
+}
+
 //
 // Kotlin Plugin compiler arguments
 // https://kotlinlang.org/docs/reference/using-gradle.html#compiler-options

diff --git a/api/buildSrc/src/main/groovy/org/pmiops/workbench/tooling/IncrementalHotSwapTask.groovy b/api/buildSrc/src/main/groovy/org/pmiops/workbench/tooling/IncrementalHotSwapTask.groovy
@@ -27,7 +27,7 @@ class IncrementalHotSwapTask extends DefaultTask {
                 String fqClassName = relativePath[0..-(".class".length() + 1)].replaceAll("/", ".")
                 printf "Hot swapping $fqClassName from $change.file.path...\n"
                 def proc = "echo redefine $fqClassName $change.file.path".execute() \
-                         | "${System.getenv('JAVA_HOME')}/bin/jdb -attach 8001".execute()
+                    | "jdb -attach 8001".execute()
                 def jdbOutput = new StringBuffer()
                 proc.consumeProcessOutput(jdbOutput, System.err)
                 proc.waitFor()

diff --git a/api/config/cdr_config_perf.json b/api/config/cdr_config_perf.json
@@ -66,6 +66,7 @@
       "name": "Synthetic Dataset in the Controlled Tier",
       "bigqueryProject": "fc-aou-cdr-perf-ct",
       "bigqueryDataset": "test_R2019q4r3",
+      "wgsBigqueryDataset": "1kg_wgs",
       "creationTime": "2021-04-20 00:00:00Z",
       "releaseNumber": 3,
       "numParticipants": 234525,

diff --git a/api/config/cdr_config_preprod.json b/api/config/cdr_config_preprod.json
@@ -63,7 +63,6 @@
     {
       "cdrVersionId": 4,
       "name": "All of Us Registered Tier Dataset v4",
-      "isDefault": true,
       "bigqueryProject": "fc-aou-cdr-preprod",
       "bigqueryDataset": "R2020Q4R3",
       "creationTime": "2020-10-20 00:00:00Z",
@@ -90,6 +89,20 @@
       "accessTier": "controlled",
       "allSamplesWgsDataBucket": "5/wgs/vcf/merged/alpha2",
       "singleSampleArrayDataBucket": "5/microarray/vcf/single_sample/"
+    },
+    {
+      "cdrVersionId": 6,
+      "name": "All of Us Registered Tier Dataset v5",
+      "isDefault": true,
+      "bigqueryProject": "fc-aou-cdr-preprod",
+      "bigqueryDataset": "R2021Q3R2",
+      "creationTime": "2021-08-10 00:00:00Z",
+      "releaseNumber": 7,
+      "numParticipants": 329070,
+      "cdrDbName": "r_2021q3_1",
+      "hasFitbitData": true,
+      "hasCopeSurveyData": true,
+      "accessTier": "registered"
     }
   ]
 }
diff --git a/api/config/cdr_config_staging.json b/api/config/cdr_config_staging.json
@@ -110,6 +110,7 @@
       "name": "Synthetic Dataset in the Controlled Tier",
       "bigqueryProject": "fc-aou-cdr-staging-ct",
       "bigqueryDataset": "test_R2019q4r3",
+      "wgsBigqueryDataset": "1kg_wgs",
       "creationTime": "2021-04-20 00:00:00Z",
       "releaseNumber": 3,
       "numParticipants": 234525,

diff --git a/api/config/config_local.json b/api/config/config_local.json
@@ -31,7 +31,8 @@
     "freeTierCostAlertThresholds": [
       0.5,
       0.75
-    ]
+    ],
+    "carahsoftEmail": "workbench-notificatio-aaaaekogch7vg3c6hi4sgyx4le@pmi-engteam.slack.com"
   },
   "auth": {
     "serviceAccountApiUsers": [
@@ -73,8 +74,7 @@
     "appEngineLocationId": "us-central"
   },
   "admin": {
-    "loginUrl": "http:\/\/localhost:4200/login",
-    "adminIdVerification": "[email protected]"
+    "loginUrl": "http:\/\/localhost:4200/login"
   },
   "mandrill": {
     "fromEmail": "[email protected]",
@@ -131,7 +131,8 @@
   },
   "accessRenewal": {
     "expiryDays": 365,
-    "expiryDaysWarningThresholds": [ 1, 3, 7, 15, 30 ]
+    "expiryDaysWarningThresholds": [ 1, 3, 7, 15, 30 ],
+    "lookbackPeriod": 330
   },
   "offlineBatch": {
     "unsafeCloudTasksForwardingHost": "http:\/\/localhost:8081",