precache: Allow experiment bitsets of any size.

components/precache/core/precache_fetcher.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "components/precache/core/precache_fetcher.h"
 
 #include <algorithm>
 #include <limits>
 #include <set>
 #include <utility>
 #include <vector>
 
 #include "base/base64.h"
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/containers/hash_tables.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/memory/ref_counted.h"
 #include "base/metrics/histogram_macros.h"
+#include "base/optional.h"
 #include "base/sha1.h"
 #include "base/strings/string_piece.h"
 #include "base/task_runner_util.h"
 #include "components/data_use_measurement/core/data_use_user_data.h"
 #include "components/precache/core/precache_database.h"
 #include "components/precache/core/precache_switches.h"
 #include "components/precache/core/proto/quota.pb.h"
 #include "components/precache/core/proto/unfinished_work.pb.h"
 #include "net/base/completion_callback.h"
 #include "net/base/escape.h"
@@ skipping 83 matching lines @@
   }
   if (!message->ParseFromString(response_string)) {
     DLOG(WARNING) << "Unable to parse proto served from "
                   << source.GetOriginalURL().spec();
     return false;
   }
   return true;
 }
 
 // Returns the resource selection bitset from the |manifest| for the given
-// |experiment_id|. By default all resource will be selected if the experiment
-// group is not found.
-uint64_t GetResourceBitset(const PrecacheManifest& manifest,
-                           uint32_t experiment_id) {
+// |experiment_id|. If the experiment group is not found, then this returns
+// nullopt, in which case all resources should be selected.
+base::Optional<std::vector<bool>> GetResourceBitset(
+    const PrecacheManifest& manifest,
+    uint32_t experiment_id) {

[pasko, 2017/02/24 13:42:33]

Do I understand correctly that each server manifest maps a finch group to a
bitset, and then you also experiment with different bitsets in the manifest
generation pipeline?

Wow. Advanced.

Did you find 64 to be limiting in some specific cases or you just want to
experiment with larger bitsets and see what happens?

[twifkak, 2017/02/24 21:20:12]

Yup.
Thanks. Or sorry. :P
Don't know for certain, but we suspect it's an issue. 64 is the number of
_unfiltered_ resources -- the more filters we experiment with, the fewer
_filtered_ resources per experiment we get. So, e.g. if we experiment with
filtering out short-lived or uncached resources, and 90% of resources are
filtered out, then we only get 6.4 resources per manifest.

In addition, with global ranking, it may be useful for some sites to have more
resources while others have less.

[pasko, 2017/02/27 18:12:16]

Ah, thank you for background. Naive question: why not applying bitsets after
filtering? Is it something historical?

[twifkak, 2017/02/27 21:46:28]

Perhaps I misunderstand your question. The bitsets are the mechanism for
client-side filtering. The idea is we want to play with different manifest
variants; it would be possible to generate and host multiple copies of the
manifests, but this is much cheaper. So let's say we want to experiment with two
versions of the manifest:

Variant 1: [a.js, b.js]
Variant 2: [a.js, c.js]

Then we'd encode that as:

{resources: [a.js, b.js, c.js],
 variants: {1: 0x3, 2: 0x5}}

Because the bitset was a fixed64, that put a hard limit on the number of
elements in 'resources' at 64, regardless of how many bits are set.

PS--The history of this feature is we added it to test the question "Should we
precache images?" We've more recently used it to test several other what-ifs,
though.

[pasko, 2017/02/28 12:21:31]

Oh, ah. Having bitsets _for_ filtering makes a lot of sense to me. Thank you.

+  base::Optional<std::vector<bool>> ret;
   if (manifest.has_experiments()) {
     const auto& resource_bitset_map =
         manifest.experiments().resources_by_experiment_group();
-    const auto& resource_bitset_it = resource_bitset_map.find(experiment_id);
-    if (resource_bitset_it != resource_bitset_map.end())
-      return resource_bitset_it->second.bitset();
+    const auto& it = resource_bitset_map.find(experiment_id);
+    if (it != resource_bitset_map.end()) {
+      if (it->second.has_bitset()) {
+        const std::string& bitset = it->second.bitset();
+        ret.emplace(bitset.size() * 8);
+        for (size_t i = 0; i < bitset.size(); ++i) {
+          for (size_t j = 0; j < 8; ++j) {
+            if ((1 << j) & bitset[i])
+              ret.value()[i * 8 + j] = true;
+          }
+        }
+      } else if (it->second.has_deprecated_bitset()) {
+        uint64_t bitset = it->second.deprecated_bitset();
+        ret.emplace(64);
+        for (int i = 0; i < 64; ++i) {
+          if ((0x1ULL << i) & bitset)
+            ret.value()[i] = true;
+        }
+      }
+    }
   }
-  return ~0ULL;
+  // Only return one variable to ensure RVO triggers.

[pasko, 2017/02/24 13:42:33]

what is RVO? Render View Observer?

[twifkak, 2017/02/24 21:20:12]

Return Value Optimization. Constructs the object on the stack frame of the
calling function, rather than constructing it here and then copying or moving it
back.

[pasko, 2017/02/27 18:12:16]

Acknowledged.

+  return ret;
 }
 
 // URLFetcherResponseWriter that ignores the response body, in order to avoid
 // the unnecessary memory usage. Use it rather than the default if you don't
 // care about parsing the response body. We use it below as a means to populate
 // the cache with requested resource URLs.
 class URLFetcherNullWriter : public net::URLFetcherResponseWriter {
  public:
   int Initialize(const net::CompletionCallback& callback) override {
     return net::OK;
@@ skipping 589 matching lines @@
   UpdateStats(source.response_bytes(), source.network_response_bytes());
   if (source.network_url_fetcher() == nullptr) {
     pool_.DeleteAll();  // Cancel any other ongoing request.
   } else {
     PrecacheManifest manifest;
 
     if (ParseProtoFromFetchResponse(*source.network_url_fetcher(), &manifest)) {
       const int32_t len =
           std::min(manifest.resource_size(),
                    unfinished_work_->config_settings().top_resources_count());
-      const uint64_t resource_bitset =
+      const base::Optional<std::vector<bool>> resource_bitset =
           GetResourceBitset(manifest, experiment_id_);
       for (int i = 0; i < len; ++i) {
-        if (((0x1ULL << i) & resource_bitset) &&
+        if ((!resource_bitset.has_value() || resource_bitset.value()[i]) &&
             manifest.resource(i).has_url()) {
           GURL url(manifest.resource(i).url());
           if (url.is_valid()) {
             double weight = ResourceWeight(
                 unfinished_work_->config_settings().resource_weight_function(),
                 manifest.resource(i).weight_ratio(), host_visits);
             if (weight >= unfinished_work_->config_settings().min_weight())
               resources_to_rank_.emplace_back(url, source.referrer(), weight);
           }
         }
@@ skipping 80 matching lines @@
       remaining = 0;
     quota_.set_remaining(
         used_bytes > quota_.remaining() ? 0U : quota_.remaining() - used_bytes);
     db_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&PrecacheDatabase::SaveQuota, precache_database_, quota_));
   }
 }
 
 }  // namespace precache