Merge "Healthcheck: proper job allocation and test." into rvc-dev am: c4ac9d2321

Change-Id: I9257064a3c0a064f2217adbe8ab5c00f26027896

1 files changed, 211 insertions, 16 deletions

diff --git a/services/incremental/test/IncrementalServiceTest.cpp b/services/incremental/test/IncrementalServiceTest.cpp
index 84ec7d3b2c24..26b5094a795a 100644
--- a/services/incremental/test/IncrementalServiceTest.cpp
+++ b/services/incremental/test/IncrementalServiceTest.cpp
@@ -22,6 +22,7 @@
 #include <gtest/gtest.h>
 #include <utils/Log.h>
 
+#include <chrono>
 #include <future>
 
 #include "IncrementalService.h"
@@ -295,9 +296,21 @@ public:
     void openMountSuccess() {
         ON_CALL(*this, openMount(_)).WillByDefault(Invoke(this, &MockIncFs::openMountForHealth));
     }
-    void waitForPendingReadsSuccess() {
+
+    // 1000ms
+    void waitForPendingReadsSuccess(uint64_t ts = 0) {
+        ON_CALL(*this, waitForPendingReads(_, _, _))
+                .WillByDefault(
+                        Invoke([ts](const Control& control, std::chrono::milliseconds timeout,
+                                    std::vector<incfs::ReadInfo>* pendingReadsBuffer) {
+                            pendingReadsBuffer->push_back({.bootClockTsUs = ts});
+                            return android::incfs::WaitResult::HaveData;
+                        }));
+    }
+
+    void waitForPendingReadsTimeout() {
         ON_CALL(*this, waitForPendingReads(_, _, _))
-                .WillByDefault(Invoke(this, &MockIncFs::waitForPendingReadsForHealth));
+                .WillByDefault(Return(android::incfs::WaitResult::Timeout));
     }
 
     static constexpr auto kPendingReadsFd = 42;
@@ -305,13 +318,6 @@ public:
         return UniqueControl(IncFs_CreateControl(-1, kPendingReadsFd, -1));
     }
 
-    WaitResult waitForPendingReadsForHealth(
-            const Control& control, std::chrono::milliseconds timeout,
-            std::vector<incfs::ReadInfo>* pendingReadsBuffer) const {
-        pendingReadsBuffer->push_back({.bootClockTsUs = 0});
-        return android::incfs::WaitResult::HaveData;
-    }
-
     RawMetadata getMountInfoMetadata(const Control& control, std::string_view path) {
         metadata::Mount m;
         m.mutable_storage()->set_id(100);
@@ -371,7 +377,7 @@ class MockJniWrapper : public JniWrapper {
 public:
     MOCK_CONST_METHOD0(initializeForCurrentThread, void());
 
-    MockJniWrapper() { EXPECT_CALL(*this, initializeForCurrentThread()).Times(3); }
+    MockJniWrapper() { EXPECT_CALL(*this, initializeForCurrentThread()).Times(2); }
 };
 
 class MockLooperWrapper : public LooperWrapper {
@@ -385,7 +391,7 @@ public:
         ON_CALL(*this, addFd(_, _, _, _, _))
                 .WillByDefault(Invoke(this, &MockLooperWrapper::storeCallback));
         ON_CALL(*this, removeFd(_)).WillByDefault(Invoke(this, &MockLooperWrapper::clearCallback));
-        ON_CALL(*this, pollAll(_)).WillByDefault(Invoke(this, &MockLooperWrapper::sleepFor));
+        ON_CALL(*this, pollAll(_)).WillByDefault(Invoke(this, &MockLooperWrapper::wait10Ms));
     }
 
     int storeCallback(int, int, int, android::Looper_callbackFunc callback, void* data) {
@@ -400,8 +406,10 @@ public:
         return 0;
     }
 
-    int sleepFor(int timeoutMillis) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(timeoutMillis));
+    int wait10Ms(int) {
+        // This is called from a loop in runCmdLooper.
+        // Sleeping for 10ms only to avoid busy looping.
+        std::this_thread::sleep_for(10ms);
         return 0;
     }
 
@@ -409,6 +417,55 @@ public:
     void* mCallbackData = nullptr;
 };
 
+class MockTimedQueueWrapper : public TimedQueueWrapper {
+public:
+    MOCK_METHOD3(addJob, void(MountId, Milliseconds, Job));
+    MOCK_METHOD1(removeJobs, void(MountId));
+    MOCK_METHOD0(stop, void());
+
+    MockTimedQueueWrapper() {
+        ON_CALL(*this, addJob(_, _, _))
+                .WillByDefault(Invoke(this, &MockTimedQueueWrapper::storeJob));
+        ON_CALL(*this, removeJobs(_)).WillByDefault(Invoke(this, &MockTimedQueueWrapper::clearJob));
+    }
+
+    void storeJob(MountId id, Milliseconds after, Job what) {
+        mId = id;
+        mAfter = after;
+        mWhat = std::move(what);
+    }
+
+    void clearJob(MountId id) {
+        if (mId == id) {
+            mAfter = {};
+            mWhat = {};
+        }
+    }
+
+    MountId mId = -1;
+    Milliseconds mAfter;
+    Job mWhat;
+};
+
+class MockStorageHealthListener : public os::incremental::BnStorageHealthListener {
+public:
+    MOCK_METHOD2(onHealthStatus, binder::Status(int32_t storageId, int32_t status));
+
+    MockStorageHealthListener() {
+        ON_CALL(*this, onHealthStatus(_, _))
+                .WillByDefault(Invoke(this, &MockStorageHealthListener::storeStorageIdAndStatus));
+    }
+
+    binder::Status storeStorageIdAndStatus(int32_t storageId, int32_t status) {
+        mStorageId = storageId;
+        mStatus = status;
+        return binder::Status::ok();
+    }
+
+    int32_t mStorageId = -1;
+    int32_t mStatus = -1;
+};
+
 class MockServiceManager : public ServiceManagerWrapper {
 public:
     MockServiceManager(std::unique_ptr<MockVoldService> vold,
@@ -416,13 +473,15 @@ public:
                        std::unique_ptr<MockIncFs> incfs,
                        std::unique_ptr<MockAppOpsManager> appOpsManager,
                        std::unique_ptr<MockJniWrapper> jni,
-                       std::unique_ptr<MockLooperWrapper> looper)
+                       std::unique_ptr<MockLooperWrapper> looper,
+                       std::unique_ptr<MockTimedQueueWrapper> timedQueue)
           : mVold(std::move(vold)),
             mDataLoaderManager(std::move(dataLoaderManager)),
             mIncFs(std::move(incfs)),
             mAppOpsManager(std::move(appOpsManager)),
             mJni(std::move(jni)),
-            mLooper(std::move(looper)) {}
+            mLooper(std::move(looper)),
+            mTimedQueue(std::move(timedQueue)) {}
     std::unique_ptr<VoldServiceWrapper> getVoldService() final { return std::move(mVold); }
     std::unique_ptr<DataLoaderManagerWrapper> getDataLoaderManager() final {
         return std::move(mDataLoaderManager);
@@ -431,6 +490,7 @@ public:
     std::unique_ptr<AppOpsManagerWrapper> getAppOpsManager() final { return std::move(mAppOpsManager); }
     std::unique_ptr<JniWrapper> getJni() final { return std::move(mJni); }
     std::unique_ptr<LooperWrapper> getLooper() final { return std::move(mLooper); }
+    std::unique_ptr<TimedQueueWrapper> getTimedQueue() final { return std::move(mTimedQueue); }
 
 private:
     std::unique_ptr<MockVoldService> mVold;
@@ -439,6 +499,7 @@ private:
     std::unique_ptr<MockAppOpsManager> mAppOpsManager;
     std::unique_ptr<MockJniWrapper> mJni;
     std::unique_ptr<MockLooperWrapper> mLooper;
+    std::unique_ptr<MockTimedQueueWrapper> mTimedQueue;
 };
 
 // --- IncrementalServiceTest ---
@@ -460,6 +521,8 @@ public:
         mJni = jni.get();
         auto looper = std::make_unique<NiceMock<MockLooperWrapper>>();
         mLooper = looper.get();
+        auto timedQueue = std::make_unique<NiceMock<MockTimedQueueWrapper>>();
+        mTimedQueue = timedQueue.get();
         mIncrementalService =
                 std::make_unique<IncrementalService>(MockServiceManager(std::move(vold),
                                                                         std::move(
@@ -467,7 +530,8 @@ public:
                                                                         std::move(incFs),
                                                                         std::move(appOps),
                                                                         std::move(jni),
-                                                                        std::move(looper)),
+                                                                        std::move(looper),
+                                                                        std::move(timedQueue)),
                                                      mRootDir.path);
         mDataLoaderParcel.packageName = "com.test";
         mDataLoaderParcel.arguments = "uri";
@@ -503,6 +567,7 @@ protected:
     NiceMock<MockAppOpsManager>* mAppOpsManager = nullptr;
     NiceMock<MockJniWrapper>* mJni = nullptr;
     NiceMock<MockLooperWrapper>* mLooper = nullptr;
+    NiceMock<MockTimedQueueWrapper>* mTimedQueue = nullptr;
     NiceMock<MockDataLoader>* mDataLoader = nullptr;
     std::unique_ptr<IncrementalService> mIncrementalService;
     TemporaryDir mRootDir;
@@ -710,6 +775,136 @@ TEST_F(IncrementalServiceTest, testStartDataLoaderRecreateOnPendingReads) {
     mLooper->mCallback(-1, -1, mLooper->mCallbackData);
 }
 
+TEST_F(IncrementalServiceTest, testStartDataLoaderUnhealthyStorage) {
+    mVold->mountIncFsSuccess();
+    mIncFs->makeFileSuccess();
+    mIncFs->openMountSuccess();
+    mVold->bindMountSuccess();
+    mDataLoaderManager->bindToDataLoaderSuccess();
+    mDataLoaderManager->getDataLoaderSuccess();
+    EXPECT_CALL(*mDataLoaderManager, bindToDataLoader(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoaderManager, unbindFromDataLoader(_)).Times(1);
+    EXPECT_CALL(*mDataLoader, create(_, _, _, _)).Times(1);
+    EXPECT_CALL(*mDataLoader, start(_)).Times(1);
+    EXPECT_CALL(*mDataLoader, destroy(_)).Times(1);
+    EXPECT_CALL(*mVold, unmountIncFs(_)).Times(2);
+    EXPECT_CALL(*mLooper, addFd(MockIncFs::kPendingReadsFd, _, _, _, _)).Times(2);
+    EXPECT_CALL(*mLooper, removeFd(MockIncFs::kPendingReadsFd)).Times(2);
+    EXPECT_CALL(*mTimedQueue, addJob(_, _, _)).Times(4);
+
+    sp<NiceMock<MockStorageHealthListener>> listener{new NiceMock<MockStorageHealthListener>};
+    NiceMock<MockStorageHealthListener>* listenerMock = listener.get();
+    EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_OK))
+            .Times(2);
+    EXPECT_CALL(*listenerMock,
+                onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_READS_PENDING))
+            .Times(1);
+    EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_BLOCKED))
+            .Times(1);
+    EXPECT_CALL(*listenerMock, onHealthStatus(_, IStorageHealthListener::HEALTH_STATUS_UNHEALTHY))
+            .Times(2);
+
+    StorageHealthCheckParams params;
+    params.blockedTimeoutMs = 10000;
+    params.unhealthyTimeoutMs = 20000;
+    params.unhealthyMonitoringMs = 30000;
+
+    using MS = std::chrono::milliseconds;
+    using MCS = std::chrono::microseconds;
+
+    const auto blockedTimeout = MS(params.blockedTimeoutMs);
+    const auto unhealthyTimeout = MS(params.unhealthyTimeoutMs);
+    const auto unhealthyMonitoring = MS(params.unhealthyMonitoringMs);
+
+    const uint64_t kFirstTimestampUs = 1000000000ll;
+    const uint64_t kBlockedTimestampUs =
+            kFirstTimestampUs - std::chrono::duration_cast<MCS>(blockedTimeout).count();
+    const uint64_t kUnhealthyTimestampUs =
+            kFirstTimestampUs - std::chrono::duration_cast<MCS>(unhealthyTimeout).count();
+
+    TemporaryDir tempDir;
+    int storageId = mIncrementalService->createStorage(tempDir.path, std::move(mDataLoaderParcel),
+                                                       IncrementalService::CreateOptions::CreateNew,
+                                                       {}, std::move(params), listener);
+    ASSERT_GE(storageId, 0);
+
+    // Healthy state, registered for pending reads.
+    ASSERT_NE(nullptr, mLooper->mCallback);
+    ASSERT_NE(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
+
+    // Looper/epoll callback.
+    mIncFs->waitForPendingReadsSuccess(kFirstTimestampUs);
+    mLooper->mCallback(-1, -1, mLooper->mCallbackData);
+
+    // Unregister from pending reads and wait.
+    ASSERT_EQ(nullptr, mLooper->mCallback);
+    ASSERT_EQ(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_READS_PENDING, listener->mStatus);
+    // Timed callback present.
+    ASSERT_EQ(storageId, mTimedQueue->mId);
+    ASSERT_GE(mTimedQueue->mAfter, blockedTimeout);
+    auto timedCallback = mTimedQueue->mWhat;
+    mTimedQueue->clearJob(storageId);
+
+    // Timed job callback for blocked.
+    mIncFs->waitForPendingReadsSuccess(kBlockedTimestampUs);
+    timedCallback();
+
+    // Still not registered, and blocked.
+    ASSERT_EQ(nullptr, mLooper->mCallback);
+    ASSERT_EQ(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_BLOCKED, listener->mStatus);
+    // Timed callback present.
+    ASSERT_EQ(storageId, mTimedQueue->mId);
+    ASSERT_GE(mTimedQueue->mAfter, 1000ms);
+    timedCallback = mTimedQueue->mWhat;
+    mTimedQueue->clearJob(storageId);
+
+    // Timed job callback for unhealthy.
+    mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
+    timedCallback();
+
+    // Still not registered, and blocked.
+    ASSERT_EQ(nullptr, mLooper->mCallback);
+    ASSERT_EQ(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
+    // Timed callback present.
+    ASSERT_EQ(storageId, mTimedQueue->mId);
+    ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
+    timedCallback = mTimedQueue->mWhat;
+    mTimedQueue->clearJob(storageId);
+
+    // One more unhealthy.
+    mIncFs->waitForPendingReadsSuccess(kUnhealthyTimestampUs);
+    timedCallback();
+
+    // Still not registered, and blocked.
+    ASSERT_EQ(nullptr, mLooper->mCallback);
+    ASSERT_EQ(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_UNHEALTHY, listener->mStatus);
+    // Timed callback present.
+    ASSERT_EQ(storageId, mTimedQueue->mId);
+    ASSERT_GE(mTimedQueue->mAfter, unhealthyMonitoring);
+    timedCallback = mTimedQueue->mWhat;
+    mTimedQueue->clearJob(storageId);
+
+    // And now healthy.
+    mIncFs->waitForPendingReadsTimeout();
+    timedCallback();
+
+    // Healthy state, registered for pending reads.
+    ASSERT_NE(nullptr, mLooper->mCallback);
+    ASSERT_NE(nullptr, mLooper->mCallbackData);
+    ASSERT_EQ(storageId, listener->mStorageId);
+    ASSERT_EQ(IStorageHealthListener::HEALTH_STATUS_OK, listener->mStatus);
+}
+
 TEST_F(IncrementalServiceTest, testSetIncFsMountOptionsSuccess) {
     mVold->mountIncFsSuccess();
     mIncFs->makeFileSuccess();