)]}'
{"etc/container-server.conf-sample":[{"author":{"_account_id":5600,"name":"Brian Cline","email":"bcline@softlayer.com","username":"briancline"},"change_message_id":"6df66591997bb6db7d15c1cc6fe85dcd9d18b46a","unresolved":false,"context_lines":[{"line_number":189,"context_line":"# processes \u003d 1"},{"line_number":190,"context_line":"#"},{"line_number":191,"context_line":"# The maximum number of threads running in each process"},{"line_number":192,"context_line":"# Different processes sync different rows of a container"},{"line_number":193,"context_line":"# threads \u003d 1"},{"line_number":194,"context_line":""},{"line_number":195,"context_line":"# Note: Put it at the beginning of the pipeline to profile all middleware. But"}],"source_content_type":"application/octet-stream","patch_set":2,"id":"fa80f949_f413d45a","line":192,"updated":"2015-11-03 22:55:07.000000000","message":"Different processes or different threads?","commit_id":"e50afdd257e6d78445582935034466d3a2753611"}],"swift/container/sync.py":[{"author":{"_account_id":5600,"name":"Brian Cline","email":"bcline@softlayer.com","username":"briancline"},"change_message_id":"6df66591997bb6db7d15c1cc6fe85dcd9d18b46a","unresolved":false,"context_lines":[{"line_number":146,"context_line":"    Daemon to sync syncable containers."},{"line_number":147,"context_line":""},{"line_number":148,"context_line":"    This is done by scanning the sync store which holds pointers to container"},{"line_number":149,"context_line":"    datadases that are configured for container sync. The daemon spawns"},{"line_number":150,"context_line":"    processes up to a configurable limit assigning each process a different"},{"line_number":151,"context_line":"    container db to sync. Each process uses a green pool of threads (of"},{"line_number":152,"context_line":"    configurable size) to sync the given container db. For each container db"}],"source_content_type":"text/x-python","patch_set":2,"id":"fa80f949_510e7235","line":149,"updated":"2015-11-03 22:55:07.000000000","message":"s/datadases/databases/  :)","commit_id":"e50afdd257e6d78445582935034466d3a2753611"},{"author":{"_account_id":5600,"name":"Brian Cline","email":"bcline@softlayer.com","username":"briancline"},"change_message_id":"6df66591997bb6db7d15c1cc6fe85dcd9d18b46a","unresolved":false,"context_lines":[{"line_number":559,"context_line":"                           self.next_sync_point.is_set() is True:"},{"line_number":560,"context_line":"                            break"},{"line_number":561,"context_line":""},{"line_number":562,"context_line":"                        # Seems redundant - TODO: remove this"},{"line_number":563,"context_line":"                        key \u003d hash_path(info[\u0027account\u0027], info[\u0027container\u0027],"},{"line_number":564,"context_line":"                                        row[\u0027name\u0027], raw_digest\u003dTrue)"},{"line_number":565,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"fa80f949_d4e4903c","line":562,"updated":"2015-11-03 22:55:07.000000000","message":"Was this note meant to be addressed before this patch gets merged, or for later?","commit_id":"e50afdd257e6d78445582935034466d3a2753611"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"c8f53103d7b4b13613b6c9295bade0dd3135082a","unresolved":false,"context_lines":[{"line_number":432,"context_line":"                rows_from +\u003d len(rows)"},{"line_number":433,"context_line":"                index \u003d 0"},{"line_number":434,"context_line":"            if rows:"},{"line_number":435,"context_line":"                row \u003d rows[index]"},{"line_number":436,"context_line":"                index +\u003d 1"},{"line_number":437,"context_line":"                yield row"},{"line_number":438,"context_line":"            else:"}],"source_content_type":"text/x-python","patch_set":6,"id":"da0c15f0_9f7700d4","line":435,"range":{"start_line":435,"start_character":27,"end_line":435,"end_character":32},"updated":"2016-03-25 09:06:59.000000000","message":"I prefer loop processing using for x in y to using index, so how about the following code?\n\n    rows_from \u003d sync_point1\n    while True:\n\n        # Retrieve rows to be processed next\n        rows \u003d broker.get_items_since(rows_from, 100)\n    \n        # Break if we consumed all rows\n        if not rows:\n            break\n\n        rows_from +\u003d len(rows)\n\n        # yield each rows\n        for row in rows:\n            yield row","commit_id":"60954947ea3e87a230b6ea1ac34d1fcec51e6269"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"c8f53103d7b4b13613b6c9295bade0dd3135082a","unresolved":false,"context_lines":[{"line_number":444,"context_line":"        In this phase each node traverses ALL rows up to sync_point1."},{"line_number":445,"context_line":"        To optimize db access we read from the db in bulks."},{"line_number":446,"context_line":"        \"\"\""},{"line_number":447,"context_line":"        rows \u003d list()"},{"line_number":448,"context_line":"        rows_from \u003d sync_point2"},{"line_number":449,"context_line":"        rows \u003d broker.get_items_since(rows_from, 100)"},{"line_number":450,"context_line":"        # Take care of special case where point2 \u003d\u003d -1"}],"source_content_type":"text/x-python","patch_set":6,"id":"da0c15f0_5ffe086c","line":447,"range":{"start_line":447,"start_character":8,"end_line":447,"end_character":21},"updated":"2016-03-25 09:06:59.000000000","message":"Do we need this?","commit_id":"60954947ea3e87a230b6ea1ac34d1fcec51e6269"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"c8f53103d7b4b13613b6c9295bade0dd3135082a","unresolved":false,"context_lines":[{"line_number":448,"context_line":"        rows_from \u003d sync_point2"},{"line_number":449,"context_line":"        rows \u003d broker.get_items_since(rows_from, 100)"},{"line_number":450,"context_line":"        # Take care of special case where point2 \u003d\u003d -1"},{"line_number":451,"context_line":"        if rows_from \u003d\u003d -1:"},{"line_number":452,"context_line":"            rows_from \u003d 0"},{"line_number":453,"context_line":"        # update the next db point to read rows from"},{"line_number":454,"context_line":"        rows_from +\u003d len(rows)"},{"line_number":455,"context_line":"        index \u003d 0"}],"source_content_type":"text/x-python","patch_set":6,"id":"da0c15f0_bf4644bd","line":452,"range":{"start_line":451,"start_character":8,"end_line":452,"end_character":25},"updated":"2016-03-25 09:06:59.000000000","message":"Is it really needed? I suppose, when we pass -1 as from parameter, it returns rows from id 0 without any error.","commit_id":"60954947ea3e87a230b6ea1ac34d1fcec51e6269"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"10367d59fec78defb96a971b5f1ad86524e30361","unresolved":false,"context_lines":[{"line_number":448,"context_line":"        rows_from \u003d sync_point2"},{"line_number":449,"context_line":"        rows \u003d broker.get_items_since(rows_from, 100)"},{"line_number":450,"context_line":"        # Take care of special case where point2 \u003d\u003d -1"},{"line_number":451,"context_line":"        if rows_from \u003d\u003d -1:"},{"line_number":452,"context_line":"            rows_from \u003d 0"},{"line_number":453,"context_line":"        # update the next db point to read rows from"},{"line_number":454,"context_line":"        rows_from +\u003d len(rows)"},{"line_number":455,"context_line":"        index \u003d 0"}],"source_content_type":"text/x-python","patch_set":6,"id":"1a122d0e_19f30275","line":452,"range":{"start_line":451,"start_character":8,"end_line":452,"end_character":25},"in_reply_to":"da0c15f0_bf4644bd","updated":"2016-04-16 16:43:45.000000000","message":"Oh, I noticed this is needed, because removing this causes wrong rows_from value in the following while loop.","commit_id":"60954947ea3e87a230b6ea1ac34d1fcec51e6269"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"c8f53103d7b4b13613b6c9295bade0dd3135082a","unresolved":false,"context_lines":[{"line_number":459,"context_line":"            if index \u003d\u003d len(rows) - 1:"},{"line_number":460,"context_line":"                # We have processed all rows"},{"line_number":461,"context_line":"                # need to read next bulk from db"},{"line_number":462,"context_line":"                rows \u003d broker.get_items_since(rows_from, 100)"},{"line_number":463,"context_line":"                if rows:"},{"line_number":464,"context_line":"                    rows_from +\u003d len(rows)"},{"line_number":465,"context_line":"                    index \u003d 0"}],"source_content_type":"text/x-python","patch_set":6,"id":"da0c15f0_ff7d5ce8","line":462,"range":{"start_line":462,"start_character":23,"end_line":462,"end_character":61},"updated":"2016-03-25 09:06:59.000000000","message":"This function is called inside and outside the while roop.\nDoes the following refactoring make sense?\n\n    rows_from \u003d sync_point2\n    next_id \u003d 0\n\n    while next_id \u003c\u003d sync_point1:\n    \n        # Retrieve rows to be processed next\n        rows \u003d broker.get_items_since(rows_from, 100)\n        rows_from +\u003d len(rows)\n\n        # yield each rows\n        for row in rows:\n            next_id \u003d row[\u0027ROWID\u0027]\n            if next_id \u003e sunc_point1:\n                break\n            yield row","commit_id":"60954947ea3e87a230b6ea1ac34d1fcec51e6269"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"506428e3f42a1c4a8d2139221bb1b8952488d68f","unresolved":false,"context_lines":[{"line_number":427,"context_line":"            if not rows:"},{"line_number":428,"context_line":"                break"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"            rows_from +\u003d len(rows)"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"            # yield each rows"},{"line_number":433,"context_line":"            for row in rows:"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a122d0e_19c142e5","line":430,"range":{"start_line":430,"start_character":12,"end_line":430,"end_character":34},"updated":"2016-04-16 16:58:07.000000000","message":"One question. What happens if there are some row ids missing because they have already got reclaimed? I\u0027m not sure that situation often happens in real cases, but I\u0027m afraid that may possibly causes wrong handling about row id.","commit_id":"d078484831782445a7fa1968c3ca39a9e27f958d"},{"author":{"_account_id":11317,"name":"Eran Rom","email":"eran@itsonlyme.name","username":"eranr"},"change_message_id":"8a70307240b55ee56c8db5c77117a8c8c1f44953","unresolved":false,"context_lines":[{"line_number":427,"context_line":"            if not rows:"},{"line_number":428,"context_line":"                break"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"            rows_from +\u003d len(rows)"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"            # yield each rows"},{"line_number":433,"context_line":"            for row in rows:"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a122d0e_5479c1fb","line":430,"range":{"start_line":430,"start_character":12,"end_line":430,"end_character":34},"in_reply_to":"1a122d0e_19c142e5","updated":"2016-04-16 18:51:12.000000000","message":"I can interpret the question in more then one way:\n1. row with ID x can be followed by a row with ID x+2 (rather then x+1). Not sure this is a problem as we are not assuming anything about the row ID other then being in ascending order.\n2. After reading many rows, some of them are reclaimed, that is, they are being deleted from the db. This implies that we are working on very old operations of objects that are no longer valid but the valid operations appear later in the db. Worst case we will fail to do the required operation, and next time we run we will not read those rows anyway","commit_id":"d078484831782445a7fa1968c3ca39a9e27f958d"},{"author":{"_account_id":9816,"name":"Takashi Kajinami","email":"kajinamit@oss.nttdata.com","username":"kajinamit"},"change_message_id":"2f80542f598646d08c661211d9c8cfc259c7aa59","unresolved":false,"context_lines":[{"line_number":427,"context_line":"            if not rows:"},{"line_number":428,"context_line":"                break"},{"line_number":429,"context_line":""},{"line_number":430,"context_line":"            rows_from +\u003d len(rows)"},{"line_number":431,"context_line":""},{"line_number":432,"context_line":"            # yield each rows"},{"line_number":433,"context_line":"            for row in rows:"}],"source_content_type":"text/x-python","patch_set":7,"id":"1a122d0e_8a121261","line":430,"range":{"start_line":430,"start_character":12,"end_line":430,"end_character":34},"in_reply_to":"1a122d0e_5479c1fb","updated":"2016-04-17 04:01:30.000000000","message":"You\u0027re right, and after some thought, now I\u0027m only afraid of the first one.\n\nWhen you enable sync on the container you have used for a long time, it may have some reclaimed rows.\nAssume that it has 200 rows, not including row id 100.(1-99, 101-201)\nIn the first step, we get rows with row id 1-99 and 101, and then rows_from becomes 100 because we\u0027ve got 100 rows. In the second step, we get rows from row id \u0027100\u0027 and get rows with row id 101-201. As a result row id 101 is duplicated.\n\nThe number of rows, which is used when calculating rows_from, shouldn\u0027t be greater than row id, so the skipping case (you wrote as 2) doesn\u0027t happen, I think.\n\nThis logic is not changed by this patch, so I\u0027m not sure we have to deal with it in this patch, but I think there had better be a possible improvement as the follow-up work.","commit_id":"d078484831782445a7fa1968c3ca39a9e27f958d"}]}