buffet: Handle 'rateLimitExceeded' errors by retrying the request later

When server replies with 403/rateLimitExceeded, treat this error as
retriable and repeat the request after some time.

Also adjusted the first retry interval to be 1 seconds instead of
100 ms (which was added in error).

BUG=chrome-os-partner:41855
TEST=`FEATURES=test emerge-link buffet`

Change-Id: I4123a770fc88ec73f368f1c1a5ecf91f1b60f9e1
Reviewed-on: https://chromium-review.googlesource.com/281773
Trybot-Ready: Alex Vakulenko <avakulenko@chromium.org>
Tested-by: Alex Vakulenko <avakulenko@chromium.org>
Reviewed-by: Vitaly Buka <vitalybuka@chromium.org>
Reviewed-by: Aaron Kemp <kemp@google.com>
Commit-Queue: Alex Vakulenko <avakulenko@chromium.org>

buffet/device_registration_info.cc

diff --git a/buffet/device_registration_info.cc b/buffet/device_registration_info.cc
index f1c84b8..779a57d 100644
--- a/buffet/device_registration_info.cc
+++ b/buffet/device_registration_info.cc
@@ -124,7 +124,7 @@
       shill_client_{shill_client} {
   cloud_backoff_policy_.reset(new chromeos::BackoffEntry::Policy{});
   cloud_backoff_policy_->num_errors_to_ignore = 0;
-  cloud_backoff_policy_->initial_delay_ms = 100;
+  cloud_backoff_policy_->initial_delay_ms = 1000;
   cloud_backoff_policy_->multiply_factor = 2.0;
   cloud_backoff_policy_->jitter_factor = 0.1;
   cloud_backoff_policy_->maximum_backoff_ms = 30000;
@@ -602,8 +602,6 @@
     return;
   }
 
-  cloud_backoff_entry_->InformOfRequest(true);
-
   chromeos::ErrorPtr error;
   auto json_resp = chromeos::http::ParseJsonResponse(response.get(), nullptr,
                                                      &error);
@@ -614,10 +612,17 @@
 
   if (!response->IsSuccessful()) {
     ParseGCDError(json_resp.get(), &error);
+    if (status_code == chromeos::http::status_code::Forbidden &&
+        error->HasError(buffet::kErrorDomainGCDServer, "rateLimitExceeded")) {
+      // If we exceeded server quota, retry the request later.
+      RetryCloudRequest(data);
+      return;
+    }
     data->error_callback.Run(error.get());
     return;
   }
 
+  cloud_backoff_entry_->InformOfRequest(true);
   SetRegistrationStatus(RegistrationStatus::kConnected);
   data->success_callback.Run(*json_resp);
 }
@@ -632,6 +637,7 @@
 
 void DeviceRegistrationInfo::RetryCloudRequest(
     const std::shared_ptr<const CloudRequestData>& data) {
+  // TODO(avakulenko): Tie connecting/connected status to XMPP channel instead.
   SetRegistrationStatus(RegistrationStatus::kConnecting);
   cloud_backoff_entry_->InformOfRequest(false);
   SendCloudRequest(data);