corona-warn-app · jakobmoellerdev · Sep 7, 2020 · Sep 2, 2020 · Sep 3, 2020 · Sep 3, 2020
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/http/WebRequestBuilder.kt
@@ -102,9 +102,12 @@ class WebRequestBuilder(
     ): File = withContext(Dispatchers.IO) {
         val fileName = "${UUID.nameUUIDFromBytes(url.toByteArray())}.zip"
         val file = File(FileStorageHelper.keyExportDirectory, fileName)
-        file.outputStream().use {
+        if (file.exists()) file.delete()
+        file.outputStream().use { fos ->
             Timber.v("Added $url to queue.")
-            distributionService.getKeyFiles(url).byteStream().copyTo(it, DEFAULT_BUFFER_SIZE)
+            distributionService.getKeyFiles(url).byteStream().use {
+                it.copyTo(fos, DEFAULT_BUFFER_SIZE)
+            }
             Timber.v("key file request successful.")
         }
         return@withContext file

diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/LocalData.kt
@@ -6,6 +6,7 @@ import de.rki.coronawarnapp.CoronaWarnApplication
 import de.rki.coronawarnapp.R
 import de.rki.coronawarnapp.risk.RiskLevel
 import de.rki.coronawarnapp.util.security.SecurityHelper.globalEncryptedSharedPreferencesInstance
+import org.joda.time.Instant
 import java.util.Date
 
 /**
@@ -18,6 +19,11 @@ object LocalData {
 
     private val TAG: String? = LocalData::class.simpleName
 
+    private const val PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS =
+        "preference_next_time_rate_limiting_unlocks"
+    private const val PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT =
+        "preference_google_api_provide_diagnosis_keys_call_count"
+
     /****************************************************
      * ONBOARDING DATA
      ****************************************************/
@@ -390,6 +396,40 @@ object LocalData {
         }
     }
 
+    var nextTimeRateLimitingUnlocks: Instant
+        get() {
+            return Instant.ofEpochMilli(
+                getSharedPreferenceInstance().getLong(
+                    PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS,
+                    0L
+                )
+            )
+        }
+        set(value) {
+            getSharedPreferenceInstance().edit(true) {
+                putLong(
+                    PREFERENCE_NEXT_TIME_RATE_LIMITING_UNLOCKS,
+                    value.millis
+                )
+            }
+        }
+
+    var googleAPIProvideDiagnosisKeysCallCount: Int
+        get() {
+            return getSharedPreferenceInstance().getInt(
+                PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT,
+                0
+            )
+        }
+        set(value) {
+            getSharedPreferenceInstance().edit(true) {
+                putInt(
+                    PREFERENCE_GOOGLE_API_PROVIDE_DIAGNOSIS_KEYS_CALL_COUNT,
+                    value
+                )
+            }
+        }
+
     /**
      * Gets the last time of successful risk level calculation as long
      * from the EncryptedSharedPrefs

diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheDao.kt
@@ -35,6 +35,9 @@ interface KeyCacheDao {
     @Query("SELECT * FROM date")
     suspend fun getAllEntries(): List<KeyCacheEntity>
 
+    @Query("SELECT * FROM date WHERE id IN (:idList)")
+    suspend fun getAllEntries(idList: List<String>): List<KeyCacheEntity>
+
     @Query("DELETE FROM date")
     suspend fun clear()
 
@@ -44,6 +47,9 @@ interface KeyCacheDao {
     @Delete
     suspend fun deleteEntry(entity: KeyCacheEntity)
 
+    @Delete
+    suspend fun deleteEntries(entities: List<KeyCacheEntity>)
+
     @Insert
     suspend fun insertEntry(keyCacheEntity: KeyCacheEntity): Long
 }
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/storage/keycache/KeyCacheRepository.kt
@@ -88,6 +88,14 @@ class KeyCacheRepository(private val keyCacheDao: KeyCacheDao) {
         keyCacheDao.clear()
     }
 
+    suspend fun clear(idList: List<String>) {
+        if (idList.isNotEmpty()) {
+            val entries = keyCacheDao.getAllEntries(idList)
+            entries.forEach { deleteFileForEntry(it) }
+            keyCacheDao.deleteEntries(entries)
+        }
+    }
+
     suspend fun getFilesFromEntries() = keyCacheDao
         .getAllEntries()
         .map { File(it.path) }

diff --git a/...rn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt b/...rn-App/src/main/java/de/rki/coronawarnapp/transaction/RetrieveDiagnosisKeysTransaction.kt
@@ -20,27 +20,30 @@
 package de.rki.coronawarnapp.transaction
 
 import com.google.android.gms.nearby.exposurenotification.ExposureConfiguration
-import de.rki.coronawarnapp.CoronaWarnApplication
 import de.rki.coronawarnapp.nearby.InternalExposureNotificationClient
 import de.rki.coronawarnapp.service.applicationconfiguration.ApplicationConfigurationService
 import de.rki.coronawarnapp.storage.FileStorageHelper
 import de.rki.coronawarnapp.storage.LocalData
-import de.rki.coronawarnapp.storage.keycache.KeyCacheRepository
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.API_SUBMISSION
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.CLOSE
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.FETCH_DATE_UPDATE
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.FILES_FROM_WEB_REQUESTS
+import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.QUOTA_CALCULATION
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.RETRIEVE_RISK_SCORE_PARAMS
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.SETUP
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.RetrieveDiagnosisKeysTransactionState.TOKEN
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.rollback
 import de.rki.coronawarnapp.transaction.RetrieveDiagnosisKeysTransaction.start
 import de.rki.coronawarnapp.util.CachedKeyFileHolder
+import de.rki.coronawarnapp.util.GoogleQuotaCalculator
+import de.rki.coronawarnapp.util.QuotaCalculator
 import de.rki.coronawarnapp.util.di.AppInjector
 import de.rki.coronawarnapp.worker.BackgroundWorkHelper
 import org.joda.time.DateTime
 import org.joda.time.DateTimeZone
+import org.joda.time.Duration
 import org.joda.time.Instant
+import org.joda.time.chrono.GJChronology
 import timber.log.Timber
 import java.io.File
 import java.util.Date
@@ -91,6 +94,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
         /** Initial Setup of the Transaction and Transaction ID Generation and Date Lock */
         SETUP,
 
+        /** calculates the Quota so that the rate limiting is caught gracefully*/
+        QUOTA_CALCULATION,
+
         /** Initialisation of the identifying token used during the entire transaction */
         TOKEN,
 
@@ -119,16 +125,29 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
     /** atomic reference for the rollback value for created files during the transaction */
     private val exportFilesForRollback = AtomicReference<List<File>>()
 
+    private val progressTowardsQuotaForRollback = AtomicReference<Int>()
+
     private val transactionScope: TransactionCoroutineScope by lazy {
         AppInjector.component.transRetrieveKeysInjection.transactionScope
     }
 
+    private const val QUOTA_RESET_PERIOD_IN_HOURS = 24
+
+    private val quotaCalculator: QuotaCalculator<Int> = GoogleQuotaCalculator(
+        incrementByAmount = 14,
+        quotaLimit = 20,
+        quotaResetPeriod = Duration.standardHours(QUOTA_RESET_PERIOD_IN_HOURS.toLong()),
+        quotaTimeZone = DateTimeZone.UTC,
+        quotaChronology = GJChronology.getInstanceUTC()
+    )
+
     suspend fun startWithConstraints() {
         val currentDate = DateTime(Instant.now(), DateTimeZone.UTC)
         val lastFetch = DateTime(
             LocalData.lastTimeDiagnosisKeysFromServerFetch(),
             DateTimeZone.UTC
         )
+
         if (LocalData.lastTimeDiagnosisKeysFromServerFetch() == null ||
             currentDate.withTimeAtStartOfDay() != lastFetch.withTimeAtStartOfDay()
         ) {
@@ -157,6 +176,18 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
          ****************************************************/
         val currentDate = executeSetup()
 
+        /****************************************************
+         * CALCULATE QUOTA FOR PROVIDE DIAGNOSIS KEYS
+         ****************************************************/
+        val hasExceededQuota = executeQuotaCalculation()
+
+        // When we are above the Quote, cancel the execution entirely
+        if (hasExceededQuota) {
+            Timber.tag(TAG).w("above quota, skipping RetrieveDiagnosisKeys")
+            executeClose()
+            return@lockAndExecute
+        }
+
         /****************************************************
          * RETRIEVE TOKEN
          ****************************************************/
@@ -199,8 +230,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
             if (TOKEN.isInStateStack()) {
                 rollbackToken()
             }
-            if (FILES_FROM_WEB_REQUESTS.isInStateStack()) {
-                rollbackFilesFromWebRequests()
+            // we reset the quota only if the submission has not happened yet
+            if (QUOTA_CALCULATION.isInStateStack() && !API_SUBMISSION.isInStateStack()) {
+                rollbackProgressTowardsQuota()
             }
         } catch (e: Exception) {
             // We handle every exception through a RollbackException to make sure that a single EntryPoint
@@ -219,10 +251,9 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
         LocalData.googleApiToken(googleAPITokenForRollback.get())
     }
 
-    private suspend fun rollbackFilesFromWebRequests() {
-        Timber.tag(TAG).v("rollback $FILES_FROM_WEB_REQUESTS")
-        KeyCacheRepository.getDateRepository(CoronaWarnApplication.getAppContext())
-            .clear()
+    private fun rollbackProgressTowardsQuota() {
+        Timber.tag(TAG).v("rollback $QUOTA_CALCULATION")
+        quotaCalculator.resetProgressTowardsQuota(progressTowardsQuotaForRollback.get())
     }
 
     /**
@@ -235,6 +266,16 @@ object RetrieveDiagnosisKeysTransaction : Transaction() {
         currentDate
     }
 
+    /**
+     * Executes the QUOTA_CALCULATION Transaction State
+     */
+    private suspend fun executeQuotaCalculation() = executeState(
+        QUOTA_CALCULATION
+    ) {
+        progressTowardsQuotaForRollback.set(quotaCalculator.getProgressTowardsQuota())
+        quotaCalculator.calculateQuota()
+    }
+
     /**
      * Executes the TOKEN Transaction State
      */

diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/CachedKeyFileHolder.kt
@@ -37,6 +37,7 @@ import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.withContext
 import timber.log.Timber
 import java.io.File
+import java.util.Collections
 import java.util.Date
 import java.util.UUID
 
@@ -97,22 +98,30 @@ object CachedKeyFileHolder {
             val deferredQueries: MutableCollection<Deferred<Any>> = mutableListOf()
             keyCache.deleteOutdatedEntries(uuidListFromServer)
             val missingDays = getMissingDaysFromDiff(serverDates)
+            val failedEntryCacheKeys = Collections.synchronizedList(mutableListOf<String>())
             if (missingDays.isNotEmpty()) {
                 // we have a date difference
                 deferredQueries.addAll(
                     missingDays
                         .map { getURLForDay(it) }
-                        .map { url -> async { url.createDayEntryForUrl() } }
+                        .map { url ->
+                            val cacheKey = url.generateCacheKeyFromString()
+                            async {
+                                try {
+                                    url.createDayEntryForUrl(cacheKey)
+                                } catch (e: Exception) {
+                                    Timber.v("failed entry: $cacheKey")
+                                    failedEntryCacheKeys.add(cacheKey)
+                                }
+                            }
+                        }
                 )
             }
             // execute the query plan
-            try {
-                deferredQueries.awaitAll()
-            } catch (e: Exception) {
-                // For an error we clear the cache to try again
-                keyCache.clear()
-                throw e
-            }
+            deferredQueries.awaitAll()
+            Timber.v("${failedEntryCacheKeys.size} failed entries ")
+            // For an error we clear the cache to try again
+            keyCache.clear(failedEntryCacheKeys)
             keyCache.getFilesFromEntries()
                 .also { it.forEach { file -> Timber.v("cached file:${file.path}") } }
         }
@@ -161,8 +170,8 @@ object CachedKeyFileHolder {
      * Creates a date entry in the Key Cache for a given String with a unique Key Name derived from the URL
      * and the URI of the downloaded File for that given key
      */
-    private suspend fun String.createDayEntryForUrl() = keyCache.createEntry(
-        this.generateCacheKeyFromString(),
+    private suspend fun String.createDayEntryForUrl(cacheKey: String) = keyCache.createEntry(
+        cacheKey,
         WebRequestBuilder.getInstance().asyncGetKeyFilesFromServer(this).toURI(),
         DAY
     )

diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/GoogleQuotaCalculator.kt
@@ -0,0 +1,62 @@
+package de.rki.coronawarnapp.util
+
+import de.rki.coronawarnapp.storage.LocalData
+import org.joda.time.Chronology
+import org.joda.time.DateTime
+import org.joda.time.DateTimeZone
+import org.joda.time.Duration
+import org.joda.time.Instant
+
+/**
+ * This Calculator class takes multiple parameters to check if the Google API
+ * can be called or the Rate Limit has been reached. The Quota is expected to reset at
+ * the start of the day in the given timeZone and Chronology
+ *
+ * @property incrementByAmount The amount of Quota Calls to increment per Call
+ * @property quotaLimit The maximum amount of Quota Calls allowed before Rate Limiting
+ * @property quotaResetPeriod The Period after which the Quota Resets
+ * @property quotaTimeZone The Timezone to work in
+ * @property quotaChronology The Chronology to work in
+ */
+class GoogleQuotaCalculator(
+    val incrementByAmount: Int,
+    val quotaLimit: Int,
+    val quotaResetPeriod: Duration,
+    val quotaTimeZone: DateTimeZone,
+    val quotaChronology: Chronology
+) : QuotaCalculator<Int> {
+    override var hasExceededQuota: Boolean = false
+
+    override fun calculateQuota(): Boolean {
+        if (Instant.now().isAfter(LocalData.nextTimeRateLimitingUnlocks)) {
+            LocalData.nextTimeRateLimitingUnlocks = DateTime
+                .now(quotaTimeZone)
+                .withChronology(quotaChronology)
+                .plus(quotaResetPeriod)
+                .withTimeAtStartOfDay()
+                .toInstant()
+            LocalData.googleAPIProvideDiagnosisKeysCallCount = 0
+        }
+
+        if (LocalData.googleAPIProvideDiagnosisKeysCallCount <= quotaLimit) {
+            LocalData.googleAPIProvideDiagnosisKeysCallCount += incrementByAmount
+        }
+
+        hasExceededQuota = LocalData.googleAPIProvideDiagnosisKeysCallCount > quotaLimit
+
+        return hasExceededQuota
+    }
+
+    override fun resetProgressTowardsQuota(newProgress: Int) {
+        if (newProgress > quotaLimit) {
+            throw IllegalArgumentException("cannot reset progress to a value higher than the quota limit")
+        }
+        if (newProgress % incrementByAmount != 0) {
+            throw IllegalArgumentException("supplied progress is no multiple of $incrementByAmount")
+        }
+        LocalData.googleAPIProvideDiagnosisKeysCallCount = newProgress
+        hasExceededQuota = false
+    }
+
+    override fun getProgressTowardsQuota(): Int = LocalData.googleAPIProvideDiagnosisKeysCallCount
+}
diff --git a/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt b/Corona-Warn-App/src/main/java/de/rki/coronawarnapp/util/QuotaCalculator.kt
@@ -0,0 +1,29 @@
+package de.rki.coronawarnapp.util
+
+/**
+ * Class to check if a Quota has been reached based on the calculation done inside
+ * the Calculator
+ *
+ */
+interface QuotaCalculator<T> {
+    val hasExceededQuota: Boolean
+
+    /**
+     * This function is called to recalculate an old quota score
+     */
+    fun calculateQuota(): Boolean
+
+    /**
+     * Reset the quota progress
+     *
+     * @param newProgress new progress towards the quota
+     */
+    fun resetProgressTowardsQuota(newProgress: T)
+
+    /**
+     * Retrieve the current progress towards the quota
+     *
+     * @return current progress count
+     */
+    fun getProgressTowardsQuota(): T
+}