)]}'
{"swift/obj/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b3b4d3701fd6b2ffe0b901dc5ec91f2ab19cab63","unresolved":false,"context_lines":[{"line_number":379,"context_line":"        if not config_true_value("},{"line_number":380,"context_line":"                request.headers.get("},{"line_number":381,"context_line":"                    \u0027X-Backend-Clean-Expiring-Object-Queue\u0027, \u0027t\u0027)):"},{"line_number":382,"context_line":"            return"},{"line_number":383,"context_line":"        delete_at \u003d normalize_delete_at_timestamp(delete_at)"},{"line_number":384,"context_line":"        updates \u003d [(None, None)]"},{"line_number":385,"context_line":""}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_79a9a0dd","line":382,"updated":"2018-01-10 23:54:27.000000000","message":"easy enough, existing behavior for upgrade","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6d377daf2de976c7baf7b2d6dd403f903d5c2333","unresolved":false,"context_lines":[{"line_number":874,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":875,"context_line":"        if orig_delete_at !\u003d new_delete_at:"},{"line_number":876,"context_line":"            if new_delete_at:"},{"line_number":877,"context_line":"                self.delete_at_update("},{"line_number":878,"context_line":"                    \u0027PUT\u0027, new_delete_at, account, container, obj, request,"},{"line_number":879,"context_line":"                    device, policy)"},{"line_number":880,"context_line":"            if orig_delete_at:"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_39c8f853","line":877,"updated":"2018-01-11 00:43:58.000000000","message":"This will be similarly affected, though, yeah?\n\nShould it be\n\n if op \u003d\u003d \u0027DELETE\u0027 and not config_true_value(...\n\nabove?","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4d05d6e085d855dce4661845c4628809e4bb3aa2","unresolved":false,"context_lines":[{"line_number":416,"context_line":"            if not config_true_value("},{"line_number":417,"context_line":"                request.headers.get("},{"line_number":418,"context_line":"                    \u0027X-Backend-Clean-Expiring-Object-Queue\u0027, \u0027t\u0027)):"},{"line_number":419,"context_line":"                return"},{"line_number":420,"context_line":""},{"line_number":421,"context_line":"            # DELETEs of old expiration data have no way of knowing what the"},{"line_number":422,"context_line":"            # old X-Delete-At-Container was at the time of the initial setting"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_dc34494d","line":419,"updated":"2018-01-11 22:19:13.000000000","message":"oh oic - I guess I should have read this method more closely\n\nmaybe it should be shorter :\\\n\ngreat catch tim - saved us!  Why didn\u0027t tests save us :\\","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2e6847cf3e630e1054b4c00cb778d7b30fbf45e1","unresolved":false,"context_lines":[{"line_number":421,"context_line":"            # DELETEs of old expiration data have no way of knowing what the"},{"line_number":422,"context_line":"            # old X-Delete-At-Container was at the time of the initial setting"},{"line_number":423,"context_line":"            # of the data, so a best guess is made here."},{"line_number":424,"context_line":"            # Worst case is a DELETE is issued now for something that doesn\u0027t"},{"line_number":425,"context_line":"            # exist there and the original data is left where it is, where"},{"line_number":426,"context_line":"            # it will be ignored when the expirer eventually tries to issue the"},{"line_number":427,"context_line":"            # object DELETE later since the X-Delete-At value won\u0027t match up."},{"line_number":428,"context_line":"            delete_at_container \u003d get_expirer_container("},{"line_number":429,"context_line":"                delete_at, self.expiring_objects_container_divisor,"},{"line_number":430,"context_line":"                account, container, obj)"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_b94ff390","line":427,"range":{"start_line":424,"start_character":14,"end_line":427,"end_character":77},"updated":"2018-01-12 17:04:43.000000000","message":"ok, so there already existed the possibility of there being no queue entry for a deleted object (if the expiry divisor was misconfigured on the object server)","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94dd2ef2c6965c30fec0d5307eff12bbc4ab363e","unresolved":false,"context_lines":[{"line_number":421,"context_line":"            # DELETEs of old expiration data have no way of knowing what the"},{"line_number":422,"context_line":"            # old X-Delete-At-Container was at the time of the initial setting"},{"line_number":423,"context_line":"            # of the data, so a best guess is made here."},{"line_number":424,"context_line":"            # Worst case is a DELETE is issued now for something that doesn\u0027t"},{"line_number":425,"context_line":"            # exist there and the original data is left where it is, where"},{"line_number":426,"context_line":"            # it will be ignored when the expirer eventually tries to issue the"},{"line_number":427,"context_line":"            # object DELETE later since the X-Delete-At value won\u0027t match up."},{"line_number":428,"context_line":"            delete_at_container \u003d get_expirer_container("},{"line_number":429,"context_line":"                delete_at, self.expiring_objects_container_divisor,"},{"line_number":430,"context_line":"                account, container, obj)"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_4acccb60","line":427,"range":{"start_line":424,"start_character":14,"end_line":427,"end_character":77},"in_reply_to":"7f96bb07_b94ff390","updated":"2018-01-12 17:30:33.000000000","message":"...or if you ever changed the divisor, yeah? I\u0027m not actually sure how that works -- I should probably read up on it.","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"73977d1a0549d07c6180b1595d428fe59c8a5bf6","unresolved":false,"context_lines":[{"line_number":346,"context_line":"        # object does not exist and then remove the row from the expirer"},{"line_number":347,"context_line":"        # queue."},{"line_number":348,"context_line":"        #"},{"line_number":349,"context_line":"        # In other words: expirer-queue updates on object DELETE are nice to"},{"line_number":350,"context_line":"        # have, but not strictly necessary for correct operation."},{"line_number":351,"context_line":"        #"},{"line_number":352,"context_line":"        # Also, each queue update results in an async_pending record, which"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_590e24e6","line":349,"range":{"start_line":349,"start_character":48,"end_line":349,"end_character":64},"updated":"2018-01-11 00:05:40.000000000","message":"Isn\u0027t this also used on object PUT? And there, we want something like n_updates_needed (instead of 2) for most of the same reasons we want it for container updates.","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"7dda22ba049b6bc78cd5c003c7b1607c0acb4b31","unresolved":false,"context_lines":[{"line_number":346,"context_line":"        # object does not exist and then remove the row from the expirer"},{"line_number":347,"context_line":"        # queue."},{"line_number":348,"context_line":"        #"},{"line_number":349,"context_line":"        # In other words: expirer-queue updates on object DELETE are nice to"},{"line_number":350,"context_line":"        # have, but not strictly necessary for correct operation."},{"line_number":351,"context_line":"        #"},{"line_number":352,"context_line":"        # Also, each queue update results in an async_pending record, which"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_b9a5e8b7","line":349,"range":{"start_line":349,"start_character":48,"end_line":349,"end_character":64},"in_reply_to":"7f96bb07_590e24e6","updated":"2018-01-11 00:22:35.000000000","message":"On object PUT, there\u0027s two distinct sets of updates.\n\nFirst, there\u0027s the object\u0027s container updates. If you PUT /v1/a/c/o, those are the updates to container c. Completely missing one of those results in an object that never appears in container listings. We take pains to avoid that.\n\nSecond, there\u0027s the expiring-object queue updates. If you miss those, then the object doesn\u0027t expire. This is not as bad as dark data, but it\u0027s still bad. That\u0027s currently not making the right number of updates, but https://review.openstack.org/#/c/524548/ is working on it.\n\nHere, we\u0027ve got an existing object, and a DELETE, POST, or (overwriting) PUT request for it. If the old object has X-Delete-At, we should try to remove its entry in the expiring-objects queue. If we completely miss that update, then when the object expirer encounters that row, it will issue some superfluous DELETE requests (that don\u0027t even write tombstones thanks to the magic of X-If-Delete-At) and then remove the entry from the expiring-objects queue.\n\nWe could, if we wanted to, skip over this expirer-queue update entirely and object expiration would continue to function properly. It would just end up making more requests than it does now.","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b3b4d3701fd6b2ffe0b901dc5ec91f2ab19cab63","unresolved":false,"context_lines":[{"line_number":355,"context_line":"        # requests from object updaters to container servers (possibly more,"},{"line_number":356,"context_line":"        # depending on retries). Thus, it is helpful to keep this number"},{"line_number":357,"context_line":"        # small."},{"line_number":358,"context_line":"        n_desired_queue_updates \u003d 2"},{"line_number":359,"context_line":"        for i in range(len(headers)):"},{"line_number":360,"context_line":"            headers[i][\u0027X-Backend-Clean-Expiring-Object-Queue\u0027] \u003d ("},{"line_number":361,"context_line":"                \u0027t\u0027 if i \u003c n_desired_queue_updates else \u0027f\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_39ab18d7","line":358,"updated":"2018-01-10 23:54:27.000000000","message":"+1","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"6d377daf2de976c7baf7b2d6dd403f903d5c2333","unresolved":false,"context_lines":[{"line_number":6272,"context_line":"        # Mock out async_update so we don\u0027t get any async_pending files."},{"line_number":6273,"context_line":"        with mock.patch.object(self.object_controller, \u0027async_update\u0027):"},{"line_number":6274,"context_line":"            resp \u003d req.get_response(self.object_controller)"},{"line_number":6275,"context_line":"        self.assertEqual(resp.status_int, 201)"},{"line_number":6276,"context_line":""},{"line_number":6277,"context_line":"        # Overwrite with a non-expiring object"},{"line_number":6278,"context_line":"        req \u003d Request.blank("}],"source_content_type":"text/x-python","patch_set":3,"id":"7f96bb07_39cc7888","line":6275,"updated":"2018-01-11 00:43:58.000000000","message":"Our mock definitely got called though, yeah? And if *this* PUT request included a \u0027X-Backend-Clean-Expiring-Object-Queue: false\u0027 header, it *wouldn\u0027t* have... which means we only have two chances to push onto the expirer queue now.\n\nSo in an EC policy such as your 15+4, when the first two replicas fail but the rest succeed, we have durable data that isn\u0027t scheduled to be cleaned up -- it\u0027ll just linger, fill up disks, and 404 :-(","commit_id":"28aba3b71ee3eea5fea28651ec52d14654216fbc"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2e6847cf3e630e1054b4c00cb778d7b30fbf45e1","unresolved":false,"context_lines":[{"line_number":5992,"context_line":"        req \u003d Request.blank("},{"line_number":5993,"context_line":"            \u0027/sda1/p/a/c/o\u0027, environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027DELETE\u0027},"},{"line_number":5994,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(delete_time),"},{"line_number":5995,"context_line":"                     \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027false\u0027,"},{"line_number":5996,"context_line":"                     \u0027X-If-Delete-At\u0027: str(delete_at_timestamp)})"},{"line_number":5997,"context_line":"        resp \u003d req.get_response(self.object_controller)"},{"line_number":5998,"context_line":"        self.assertEqual(resp.status_int, 204)"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_7c871911","line":5995,"range":{"start_line":5995,"start_character":63,"end_line":5995,"end_character":68},"updated":"2018-01-12 17:04:43.000000000","message":"nit: the tests always use \u0027false\u0027 but the real code always sends \u0027f\u0027 (or \u0027t\u0027)","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2e6847cf3e630e1054b4c00cb778d7b30fbf45e1","unresolved":false,"context_lines":[{"line_number":6279,"context_line":"            \u0027/sda1/p/a/c/o\u0027,"},{"line_number":6280,"context_line":"            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":6281,"context_line":"            headers\u003d{\u0027X-Timestamp\u0027: normalize_timestamp(overwrite_time),"},{"line_number":6282,"context_line":"                     \u0027X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027false\u0027,"},{"line_number":6283,"context_line":"                     \u0027Content-Length\u0027: \u00279\u0027,"},{"line_number":6284,"context_line":"                     \u0027Content-Type\u0027: \u0027application/octet-stream\u0027})"},{"line_number":6285,"context_line":"        req.body \u003d \u0027new stuff\u0027"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_7f814b4a","line":6282,"range":{"start_line":6282,"start_character":62,"end_line":6282,"end_character":69},"updated":"2018-01-12 17:04:43.000000000","message":"I don\u0027t see any tests for \n\n  X-Backend-Clean-Expiring-Object-Queue\u0027: \u0027true\u0027","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"94dd2ef2c6965c30fec0d5307eff12bbc4ab363e","unresolved":false,"context_lines":[{"line_number":6344,"context_line":"            for filename in filenames:"},{"line_number":6345,"context_line":"                async_pendings.append(os.path.join(dirpath, filename))"},{"line_number":6346,"context_line":""},{"line_number":6347,"context_line":"        self.assertEqual(len(async_pendings), 1)"},{"line_number":6348,"context_line":""},{"line_number":6349,"context_line":"        async_pending_ops \u003d []"},{"line_number":6350,"context_line":"        for pending_file in async_pendings:"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_ff881472","line":6347,"updated":"2018-01-12 17:30:33.000000000","message":"I\u0027d kinda rather drop this, and have the\n\n self.assertEqual(async_pending_ops, [\u0027PUT\u0027])\n\nenforce the length check... though I can\u0027t help but feel like it might make more sense to mock out async_update again and just check its calls...","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"2e6847cf3e630e1054b4c00cb778d7b30fbf45e1","unresolved":false,"context_lines":[{"line_number":468,"context_line":"        counts \u003d {True: 0, False: 0, None: 0}"},{"line_number":469,"context_line":"        for headers in captured_headers:"},{"line_number":470,"context_line":"            v \u003d headers.get(\u0027X-Backend-Clean-Expiring-Object-Queue\u0027)"},{"line_number":471,"context_line":"            norm_v \u003d None if v is None else utils.config_true_value(v)"},{"line_number":472,"context_line":"            counts[norm_v] +\u003d 1"},{"line_number":473,"context_line":""},{"line_number":474,"context_line":"        max_queue_updates \u003d 2"}],"source_content_type":"text/x-python","patch_set":4,"id":"7f96bb07_5f6d6f02","line":471,"range":{"start_line":471,"start_character":44,"end_line":471,"end_character":70},"updated":"2018-01-12 17:04:43.000000000","message":"nit: this is subtle, but using config_true_value here in the test, rather than asserting the actual strings sent, could make it harder to spot future regressions. \n\ne.g. if I change the obj server check to:\n\n              if request.headers.get(\n                    \u0027X-Backend-Clean-Expiring-Object-Queue\u0027, \u0027t\u0027) \u003d\u003d \u0027false\u0027:\n                return\n\nthen the system breaks but no test fails.\n\ntest_server.py asserts that \u0027false\u0027 works, and it might be more immediately apparent what is broken if here we explicitly asserted that \u0027f\u0027 is sent.","commit_id":"48da3c1ed783a2b69cc74b02e8fd45e9d36cf80a"}]}