)]}'
{"bin/swift-object-auditor":[{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"5214e598a4222264b6c1db2ead4b27bdfed1fe62","unresolved":false,"context_lines":[{"line_number":25,"context_line":"                      help\u003d\u0027Audit only zero byte files at specified files/sec\u0027)"},{"line_number":26,"context_line":"    parser.add_option(\u0027-d\u0027, \u0027--devices\u0027,"},{"line_number":27,"context_line":"                      help\u003d\u0027Replicate only given devices. \u0027"},{"line_number":28,"context_line":"                           \u0027Comma-separated list\u0027)"},{"line_number":29,"context_line":"    conf_file, options \u003d parse_options(parser\u003dparser, once\u003dTrue)"},{"line_number":30,"context_line":"    run_daemon(ObjectAuditor, conf_file, **options)"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"AAAAUX%2F%2F0vk%3D","line":28,"updated":"2014-02-10 15:56:57.000000000","message":"\"Replicate\" isn\u0027t the right word there. \u003d )","commit_id":"043230e55bb32b81fed581c21da1ccb625e65755"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"6d207c19fbf2c95b6f677e307bf161a7db95b04a","unresolved":false,"context_lines":[{"line_number":25,"context_line":"                      help\u003d\u0027Audit only zero byte files at specified files/sec\u0027)"},{"line_number":26,"context_line":"    parser.add_option(\u0027-d\u0027, \u0027--devices\u0027,"},{"line_number":27,"context_line":"                      help\u003d\u0027Replicate only given devices. \u0027"},{"line_number":28,"context_line":"                           \u0027Comma-separated list\u0027)"},{"line_number":29,"context_line":"    conf_file, options \u003d parse_options(parser\u003dparser, once\u003dTrue)"},{"line_number":30,"context_line":"    run_daemon(ObjectAuditor, conf_file, **options)"}],"source_content_type":"application/octet-stream","patch_set":10,"id":"AAAAUX%2F%2FkHg%3D","line":28,"in_reply_to":"AAAAUX%2F%2F0vk%3D","updated":"2014-02-12 10:13:05.000000000","message":"Ooops - fixed in patch set 11","commit_id":"043230e55bb32b81fed581c21da1ccb625e65755"}],"bin/swift-recon":[{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"ecb5c600ed722371f59ed48c50a2712746db5510","unresolved":false,"context_lines":[{"line_number":488,"context_line":"                  (self._ptime(low), self._ptime(high), self._ptime(average))"},{"line_number":489,"context_line":"        print \"\u003d\" * 79"},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"    def object_auditor_get_value(self, key, recon_entry):"},{"line_number":492,"context_line":"        \"\"\""},{"line_number":493,"context_line":"        Generator that yields all values for given key in a recon cache entry."},{"line_number":494,"context_line":"        This is for use with object auditor recon cache entries.  If the"}],"source_content_type":"application/octet-stream","patch_set":8,"id":"AAAAUH%2F9y6g%3D","line":491,"updated":"2014-02-03 21:11:29.000000000","message":"Maybe name this something like nested_get_value? It looks fairly generic. Just a thought, not a requirement.","commit_id":"ffa3d0bb6e9be4b9e3bde17f5fe7d89c724078df"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"abafe2d483e7bffdbe8134e1016b4cebf3bd4f6e","unresolved":false,"context_lines":[{"line_number":488,"context_line":"                  (self._ptime(low), self._ptime(high), self._ptime(average))"},{"line_number":489,"context_line":"        print \"\u003d\" * 79"},{"line_number":490,"context_line":""},{"line_number":491,"context_line":"    def object_auditor_get_value(self, key, recon_entry):"},{"line_number":492,"context_line":"        \"\"\""},{"line_number":493,"context_line":"        Generator that yields all values for given key in a recon cache entry."},{"line_number":494,"context_line":"        This is for use with object auditor recon cache entries.  If the"}],"source_content_type":"application/octet-stream","patch_set":8,"id":"AAAAUH%2F9uwQ%3D","line":491,"in_reply_to":"AAAAUH%2F9y6g%3D","updated":"2014-02-04 15:53:52.000000000","message":"Done in patch set 9","commit_id":"ffa3d0bb6e9be4b9e3bde17f5fe7d89c724078df"}],"doc/source/deployment_guide.rst":[{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"9a6a5ffa69d6e0599aa329767f449d356d81c4f0","unresolved":false,"context_lines":[{"line_number":500,"context_line":"files_per_second    20              Maximum files audited per second. Should"},{"line_number":501,"context_line":"                                    be tuned according to individual system"},{"line_number":502,"context_line":"                                    specs. 0 is unlimited."},{"line_number":503,"context_line":"bytes_per_second    10000000        Maximum bytes audited per second. Should"},{"line_number":504,"context_line":"                                    be tuned according to individual system"},{"line_number":505,"context_line":"                                    specs. 0 is unlimited."},{"line_number":506,"context_line":"concurrency         1               The number of parallel processes to use"}],"source_content_type":"text/x-rst","patch_set":4,"id":"AAAAUH%2F%2B5GY%3D","line":503,"updated":"2014-01-21 01:39:02.000000000","message":"Are the files_per_second and bytes_per_second limits per auditor process, or are they global limits and each auditor gets a share?\n\n(I cheated and looked at the code for the answer, but that was my first question upon looking here. Would you add that information to the doc?)","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"29e2e7cb1a1f9c28bb8ab9389ea71d77ebc7c297","unresolved":false,"context_lines":[{"line_number":500,"context_line":"files_per_second    20              Maximum files audited per second. Should"},{"line_number":501,"context_line":"                                    be tuned according to individual system"},{"line_number":502,"context_line":"                                    specs. 0 is unlimited."},{"line_number":503,"context_line":"bytes_per_second    10000000        Maximum bytes audited per second. Should"},{"line_number":504,"context_line":"                                    be tuned according to individual system"},{"line_number":505,"context_line":"                                    specs. 0 is unlimited."},{"line_number":506,"context_line":"concurrency         1               The number of parallel processes to use"}],"source_content_type":"text/x-rst","patch_set":4,"id":"AAAAUH%2F%2Bw88%3D","line":503,"in_reply_to":"AAAAUH%2F%2B5GY%3D","updated":"2014-01-22 11:41:00.000000000","message":"Done","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"}],"swift/common/utils.py":[{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"4dbd67eec51abd3d900e2a6da12f619d0e16a731","unresolved":false,"context_lines":[{"line_number":2083,"context_line":"    \"\"\""},{"line_number":2084,"context_line":"    Function that will check if item is a dict, and if so put it under"},{"line_number":2085,"context_line":"    cache_entry[key].  We use nested recon cache entries when the object"},{"line_number":2086,"context_line":"    auditor runs in parallel or else in \u0027once\u0027 mode with a specified i"},{"line_number":2087,"context_line":"    subset of devices."},{"line_number":2088,"context_line":"    \"\"\""},{"line_number":2089,"context_line":"    if isinstance(item, dict):"}],"source_content_type":"text/x-python","patch_set":14,"id":"AAAAXH%2F%2FtsM%3D","line":2086,"updated":"2014-04-18 00:54:39.000000000","message":"The \u0027i\u0027 seems superfluous.","commit_id":"c23f5307dd84da0ad27d81749558e31eb047dc30"}],"swift/obj/auditor.py":[{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"930b9abf6ee1281c6e79ad2270a3054f96ed3ae1","unresolved":false,"context_lines":[{"line_number":267,"context_line":"                    parallel_proc \u003d self.concurrency + 1 if \\"},{"line_number":268,"context_line":"                        self.conf_zero_byte_fps else self.concurrency"},{"line_number":269,"context_line":"                    device_list \u003d listdir(self.devices)"},{"line_number":270,"context_line":"                    shuffle(device_list)"},{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F9A2E%3D","line":270,"updated":"2013-12-17 17:48:45.000000000","message":"I\u0027m not sure about this. Let\u0027s assume you have 1000 devices and 4 processes; now it can take a very long time until all devices are replicated because it might happen that the shuffling and looping over the shuffled list might not catch all devices in a reasonable time.\n\nI think the devices should be shuffled before the \"while True\" and then use a temporary device list here, removing already replicated devices. Once the list is empty, restart with all devices. Does that make sense?","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"bf546149c02a1454eb4d70ec07a4b2e264e2dc44","unresolved":false,"context_lines":[{"line_number":267,"context_line":"                    parallel_proc \u003d self.concurrency + 1 if \\"},{"line_number":268,"context_line":"                        self.conf_zero_byte_fps else self.concurrency"},{"line_number":269,"context_line":"                    device_list \u003d listdir(self.devices)"},{"line_number":270,"context_line":"                    shuffle(device_list)"},{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAUH%2F%2F7lQ%3D","line":270,"in_reply_to":"AAAAT3%2F%2F2Cs%3D","updated":"2014-01-06 16:33:25.000000000","message":"Hi Christian,\n\nI think that there is some confusion here.  Let me outline how this parallel code works:\n\n1.  The master process shuffles the list of devices\n\n2.  The master process loops though the shuffled list\n\n3.  For each device in the list, the device-name is passed to a child process for auditing.  Only parallel_proc child processes are in operation at any given time.  If this limit is hit, the master waits until a child has exited before forking a new process.  The pids of the child processes in operation at any give time are stored in the pids array\n\n4.  When all devices have been assigned to child processes the master waits for all child processes in operation to finish.\n\n5.  The master sleeps until the next auditing cycle is to begin.\n\nThe code already ensures that each device is audited once in every cycle, regardless of the value of parallel proc.\n\nWith regard to your first comment about \"even-ness\" of distribution of devices and the code contained therein, basically what you are recording on each pass through is the last 4 devices from a list that is randomly shuffled on each pass.  So in effect you are randomly sampling 4 devices from the list.  The statistics of the resulting distribution of devices are governed by the Poisson distribution (with a mean equal to the parallel_proc variable in your code).  You can calculate the \"even-ness\" of the distribution (in other words the mean deviation about the mean) of devices exactly.  This is how it varies by parallel_proc:\n\nparallel_proc  mean deviation about the mean   ratio\n     4                  1.56                   0.3907\n    12                  2.74                   0.2287\n    20                  3.55                   0.1777\n    28                  4.21                   0.1503\n    36                  4.78                   0.1327\n    44                  5.28                   0.1201\n    52                  5.74                   0.1105\n    60                  6.17                   0.1029\n    68                  6.57                   0.0966\n    76                  6.95                   0.0914\n    84                  7.31                   0.0870\n    92                  7.65                   0.0831\n   100                  7.97                   0.0797\n\nYou\u0027ll note that the \"even-ness\" of the distribution increases as parallel_proc increases.  Also note that it doesn\u0027t matter how many times you shuffle, the statistics will be governed by the Poisson distribution.\n\nHowever note the point that I make above; in each audit cycle every device will be audited once and just once although the order in which they are done will vary.  That is the point of shuffling the list of devices between each auditing cycle.\n\nI hope this helps clarify things.","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"f335b85dde0dfb22aeeca8e026635368cad23e61","unresolved":false,"context_lines":[{"line_number":267,"context_line":"                    parallel_proc \u003d self.concurrency + 1 if \\"},{"line_number":268,"context_line":"                        self.conf_zero_byte_fps else self.concurrency"},{"line_number":269,"context_line":"                    device_list \u003d listdir(self.devices)"},{"line_number":270,"context_line":"                    shuffle(device_list)"},{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAT3%2F%2F2Cs%3D","line":270,"in_reply_to":"AAAATn%2F88N8%3D","updated":"2013-12-30 10:07:56.000000000","message":"It\u0027s not a big problem if you have only a few devices per process, but not everyone will use so few devices per replication process. I think shuffling might not be required at all: you could copy the list of devices to a temporary one and .pop() devices of the list untils it\u0027s empty. If it\u0027s empty (all devices replicated once), you copy againg from original list of devices.\nThat ensures that all devices are replicated evenly, no matter how many processes are used.\nI created a small sample to illustrate this: http://paste.openstack.org/show/58492/","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"f967e65ea97b2e5a6f45f4186d10f3b301b67e8a","unresolved":false,"context_lines":[{"line_number":267,"context_line":"                    parallel_proc \u003d self.concurrency + 1 if \\"},{"line_number":268,"context_line":"                        self.conf_zero_byte_fps else self.concurrency"},{"line_number":269,"context_line":"                    device_list \u003d listdir(self.devices)"},{"line_number":270,"context_line":"                    shuffle(device_list)"},{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F88N8%3D","line":270,"in_reply_to":"AAAATn%2F9A2E%3D","updated":"2013-12-20 11:39:41.000000000","message":"Hi Christian,\n\nFirst of all thanks for taking the time to review the code.  Let me point out that in practice we\u0027ll have a much higher ratio of processes to devices than you have in your example.  Something like 10 devices per process, or even less.  We parallelised the auditor precisely to ensure that the list of all devices is traversed in a reasonable time.  Does that answer your concern?","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"f7f0916dddddb0990fd881fe8d689b3adaa519c5","unresolved":false,"context_lines":[{"line_number":267,"context_line":"                    parallel_proc \u003d self.concurrency + 1 if \\"},{"line_number":268,"context_line":"                        self.conf_zero_byte_fps else self.concurrency"},{"line_number":269,"context_line":"                    device_list \u003d listdir(self.devices)"},{"line_number":270,"context_line":"                    shuffle(device_list)"},{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAAUH%2F%2F4y8%3D","line":270,"in_reply_to":"AAAAUH%2F%2F7lQ%3D","updated":"2014-01-07 07:45:41.000000000","message":"Hi Eamonn, \n\nthanks for your very detailed response. In fact I misunderstood what was happening in line 274 (point 3 in your explanation). \nNow this makes a lot more sense to me; I\u0027ll need to test the code again but will remove my -1 in the meantime.\n\nThanks,\n\nChristian","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"930b9abf6ee1281c6e79ad2270a3054f96ed3ae1","unresolved":false,"context_lines":[{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"},{"line_number":274,"context_line":"                            pids.remove(os.wait()[0])"},{"line_number":275,"context_line":"                        pids.append(self.fork_child(**kwargs))"},{"line_number":276,"context_line":"                while pids:"},{"line_number":277,"context_line":"                    pids.remove(os.wait()[0])"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F9A8M%3D","line":274,"updated":"2013-12-17 17:48:45.000000000","message":"This line is never executed in the test, but I think it is important to get tested as well, especially if len(devices) \u003e\u003e concurrency","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"f967e65ea97b2e5a6f45f4186d10f3b301b67e8a","unresolved":false,"context_lines":[{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"},{"line_number":274,"context_line":"                            pids.remove(os.wait()[0])"},{"line_number":275,"context_line":"                        pids.append(self.fork_child(**kwargs))"},{"line_number":276,"context_line":"                while pids:"},{"line_number":277,"context_line":"                    pids.remove(os.wait()[0])"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F8yCo%3D","line":274,"in_reply_to":"AAAATn%2F9A8M%3D","updated":"2013-12-20 11:39:41.000000000","message":"Agreed - see updated patch","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b48a809d7ebe714e76d327edd654797df3b5e926","unresolved":false,"context_lines":[{"line_number":256,"context_line":"            else:"},{"line_number":257,"context_line":"                pids \u003d []"},{"line_number":258,"context_line":"                if self.conf_zero_byte_fps:"},{"line_number":259,"context_line":"                    pids.append(self.fork_child(zero_byte_fps\u003dTrue, **kwargs))"},{"line_number":260,"context_line":"                if self.concurrency \u003d\u003d 1:"},{"line_number":261,"context_line":"                    # Audit all devices in 1 process"},{"line_number":262,"context_line":"                    pids.append(self.fork_child(**kwargs))"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2F4Ns%3D","line":259,"updated":"2014-01-07 10:54:16.000000000","message":"By what means do you ensure only one zbf child is forked?  What if a zbf pid is already in the list?","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"faa5316777281320d1bf9d4a4caad26519f0b2b0","unresolved":false,"context_lines":[{"line_number":256,"context_line":"            else:"},{"line_number":257,"context_line":"                pids \u003d []"},{"line_number":258,"context_line":"                if self.conf_zero_byte_fps:"},{"line_number":259,"context_line":"                    pids.append(self.fork_child(zero_byte_fps\u003dTrue, **kwargs))"},{"line_number":260,"context_line":"                if self.concurrency \u003d\u003d 1:"},{"line_number":261,"context_line":"                    # Audit all devices in 1 process"},{"line_number":262,"context_line":"                    pids.append(self.fork_child(**kwargs))"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2Fm3s%3D","line":259,"in_reply_to":"AAAAUH%2F%2F4Ns%3D","updated":"2014-01-10 14:55:59.000000000","message":"See patch-set 3","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b48a809d7ebe714e76d327edd654797df3b5e926","unresolved":false,"context_lines":[{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"},{"line_number":274,"context_line":"                            pids.remove(os.wait()[0])"},{"line_number":275,"context_line":"                        pids.append(self.fork_child(**kwargs))"},{"line_number":276,"context_line":"                while pids:"},{"line_number":277,"context_line":"                    pids.remove(os.wait()[0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2F4Ng%3D","line":274,"updated":"2014-01-07 10:54:16.000000000","message":"This will block until an entire device is finished, this will delay the restart of zbf scanner.","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"faa5316777281320d1bf9d4a4caad26519f0b2b0","unresolved":false,"context_lines":[{"line_number":271,"context_line":"                    for device in device_list:"},{"line_number":272,"context_line":"                        kwargs[\u0027device_dir\u0027] \u003d [device]"},{"line_number":273,"context_line":"                        while len(pids) \u003e\u003d parallel_proc:"},{"line_number":274,"context_line":"                            pids.remove(os.wait()[0])"},{"line_number":275,"context_line":"                        pids.append(self.fork_child(**kwargs))"},{"line_number":276,"context_line":"                while pids:"},{"line_number":277,"context_line":"                    pids.remove(os.wait()[0])"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2Fm3Y%3D","line":274,"in_reply_to":"AAAAUH%2F%2F4Ng%3D","updated":"2014-01-10 14:55:59.000000000","message":"See patch-set 3","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"9a6a5ffa69d6e0599aa329767f449d356d81c4f0","unresolved":false,"context_lines":[{"line_number":235,"context_line":"            return pid"},{"line_number":236,"context_line":"        else:"},{"line_number":237,"context_line":"            signal.signal(signal.SIGTERM, signal.SIG_DFL)"},{"line_number":238,"context_line":"            patcher.monkey_patch(all\u003dFalse, socket\u003dTrue)"},{"line_number":239,"context_line":"            if zero_byte_fps:"},{"line_number":240,"context_line":"                kwargs[\u0027zero_byte_fps\u0027] \u003d self.conf_zero_byte_fps"},{"line_number":241,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2B5Fs%3D","line":238,"updated":"2014-01-21 01:39:02.000000000","message":"I\u0027m not sure why this is necessary; to the best of my knowledge, the auditor doesn\u0027t engage in any socket IO, with the possible exception of logging.","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"29e2e7cb1a1f9c28bb8ab9389ea71d77ebc7c297","unresolved":false,"context_lines":[{"line_number":235,"context_line":"            return pid"},{"line_number":236,"context_line":"        else:"},{"line_number":237,"context_line":"            signal.signal(signal.SIGTERM, signal.SIG_DFL)"},{"line_number":238,"context_line":"            patcher.monkey_patch(all\u003dFalse, socket\u003dTrue)"},{"line_number":239,"context_line":"            if zero_byte_fps:"},{"line_number":240,"context_line":"                kwargs[\u0027zero_byte_fps\u0027] \u003d self.conf_zero_byte_fps"},{"line_number":241,"context_line":"            try:"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2Bw8E%3D","line":238,"in_reply_to":"AAAAUH%2F%2B5Fs%3D","updated":"2014-01-22 11:41:00.000000000","message":"You\u0027re absolutely correct - line removed from Patch Set 5","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"6c56e5479dffd01ea7d6c02b161c7ed17337c638","unresolved":false,"context_lines":[{"line_number":274,"context_line":"                shuffle(device_list)"},{"line_number":275,"context_line":"                for device in device_list:"},{"line_number":276,"context_line":"                    pid \u003d None"},{"line_number":277,"context_line":"                    while len(pids) \u003e\u003d parallel_proc:"},{"line_number":278,"context_line":"                        pid \u003d os.wait()[0]"},{"line_number":279,"context_line":"                        pids.remove(pid)"},{"line_number":280,"context_line":"                    # ZBF scanner must be restarted as soon as it finishes"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAUH%2F%2BWb0%3D","line":277,"updated":"2014-01-27 19:43:23.000000000","message":"Maybe make this \"if\" instead of \"while\"?\n\nSeems like right now you can get unlucky. If two children exit simultaneously, you might call os.wait() here twice, and if the *first* reaped child is your ZBF auditor, you\u0027ll never restart it; by the time you leave this loop, \"pid\" is the pid of the non-ZBF child, and so the conditional on line 281 is never true.","commit_id":"8a5ce23f7398b823292d412d57cb6d7b21f788f4"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"36b6d8f5cb5c64dc06d12b842622c351fa659b8e","unresolved":false,"context_lines":[{"line_number":274,"context_line":"                shuffle(device_list)"},{"line_number":275,"context_line":"                for device in device_list:"},{"line_number":276,"context_line":"                    pid \u003d None"},{"line_number":277,"context_line":"                    while len(pids) \u003e\u003d parallel_proc:"},{"line_number":278,"context_line":"                        pid \u003d os.wait()[0]"},{"line_number":279,"context_line":"                        pids.remove(pid)"},{"line_number":280,"context_line":"                    # ZBF scanner must be restarted as soon as it finishes"}],"source_content_type":"text/x-python","patch_set":6,"id":"AAAAUH%2F%2BQvE%3D","line":277,"in_reply_to":"AAAAUH%2F%2BWb0%3D","updated":"2014-01-28 15:45:32.000000000","message":"Yes, that could definitely happen.  I\u0027ve changed the \"while\" to an \"if\" which now checks \"if len(pids) \u003d\u003d parallel_proc\".  So each loop traversal only culls one slave process.  While I was working on this change, I uncovered another bug which meant that some devices in device_list might be skipped for auditing.  To fix that bug, I\u0027ve replaced the \"for device in device_list\" loop control with a \"while device_list:\" control; then within the loop I pop a device off device_list and hand it to a child process if I don\u0027t have to restart the ZBF process.\n\nSee patch set 7","commit_id":"8a5ce23f7398b823292d412d57cb6d7b21f788f4"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"a12405595087b2f3eecffcb0c176add17841fda7","unresolved":false,"context_lines":[{"line_number":67,"context_line":"    def audit_all_objects(self, mode\u003d\u0027once\u0027, device_dirs\u003dNone):"},{"line_number":68,"context_line":"        if device_dirs:"},{"line_number":69,"context_line":"            self.auditor_type \u003d \"%s - parallel, devices: %s\" % ("},{"line_number":70,"context_line":"                self.auditor_type, \u0027, \u0027.join(device_dirs))"},{"line_number":71,"context_line":"        self.logger.info(_(\u0027Begin object audit \"%s\" mode (%s)\u0027) %"},{"line_number":72,"context_line":"                         (mode, self.auditor_type))"},{"line_number":73,"context_line":"        begin \u003d reported \u003d time.time()"}],"source_content_type":"text/x-python","patch_set":7,"id":"AAAAUH%2F%2BOI0%3D","line":70,"updated":"2014-01-30 01:54:17.000000000","message":"I\u0027m a little suspicious here... seems like if you run the auditor with --once, and device_dirs\u003d[sda, sdb], then you\u0027ll always report as \"parallel\" here regardless of the actual concurrency.","commit_id":"0f728c13056d08aa22642004a6a9b403ff4c6b10"},{"author":{"_account_id":2696,"name":"Darrell Bishop","email":"dbishop@nvidia.com","username":"darrellb"},"change_message_id":"1f7c198473f67051fb06dd24d016c5122e01be16","unresolved":false,"context_lines":[{"line_number":70,"context_line":"            return {top_level_key: item}"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"    def audit_all_objects(self, mode\u003d\u0027once\u0027, device_dirs\u003dNone):"},{"line_number":73,"context_line":"        description \u003d (\u0027 - %sdevices: %s\u0027 % (\u0027parallel, \u0027 if self.auditor_type"},{"line_number":74,"context_line":"                       \u003d\u003d \u0027ALL\u0027 else \u0027\u0027, \u0027, \u0027.join(device_dirs))) \\"},{"line_number":75,"context_line":"            if device_dirs else \u0027\u0027"},{"line_number":76,"context_line":"        self.logger.info(_(\u0027Begin object audit \"%s\" mode (%s%s)\u0027) %"}],"source_content_type":"text/x-python","patch_set":10,"id":"AAAAUX%2F%2F0g0%3D","line":73,"updated":"2014-02-12 06:33:17.000000000","message":"If anyone actually cared about localized log messages, they would be sad that \"parallel\", here, would not appear in the correct language.\n\nFor them, this would be better:\n\n  description \u003d \u0027\u0027\n  if device_dirs:\n      device_dir_str \u003d \u0027, \u0027.join(device_dirs)\n      if self.auditor_type \u003d\u003d \u0027ALL\u0027:\n          description \u003d _(\u0027...parallel...\u0027) % device_dir_str\n      else:\n          description \u003d _(\u0027...\u0027) % device_dir_str\n\nThen all the log lines that insert \"description\" will be consistently localized instead of ending up with a schizophrenic multi-language message.","commit_id":"043230e55bb32b81fed581c21da1ccb625e65755"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"6d207c19fbf2c95b6f677e307bf161a7db95b04a","unresolved":false,"context_lines":[{"line_number":70,"context_line":"            return {top_level_key: item}"},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"    def audit_all_objects(self, mode\u003d\u0027once\u0027, device_dirs\u003dNone):"},{"line_number":73,"context_line":"        description \u003d (\u0027 - %sdevices: %s\u0027 % (\u0027parallel, \u0027 if self.auditor_type"},{"line_number":74,"context_line":"                       \u003d\u003d \u0027ALL\u0027 else \u0027\u0027, \u0027, \u0027.join(device_dirs))) \\"},{"line_number":75,"context_line":"            if device_dirs else \u0027\u0027"},{"line_number":76,"context_line":"        self.logger.info(_(\u0027Begin object audit \"%s\" mode (%s%s)\u0027) %"}],"source_content_type":"text/x-python","patch_set":10,"id":"AAAAUX%2F%2FkH8%3D","line":73,"in_reply_to":"AAAAUX%2F%2F0g0%3D","updated":"2014-02-12 10:13:05.000000000","message":"Done - see patch-set 11","commit_id":"043230e55bb32b81fed581c21da1ccb625e65755"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":63,"context_line":"            [(s, 0) for s in self.stats_sizes + [\u0027OVER\u0027]])"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def create_recon_nested_dict(self, top_level_key, device_list, item):"},{"line_number":66,"context_line":"        if device_list:"},{"line_number":67,"context_line":"            device_key \u003d \u0027\u0027.join(device_list)"},{"line_number":68,"context_line":"            return {top_level_key: {device_key: item}}"},{"line_number":69,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fh3g%3D","line":66,"updated":"2014-02-12 15:42:27.000000000","message":"Should the devices be sorted so the key is the same each time the device list has the same devices?","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":63,"context_line":"            [(s, 0) for s in self.stats_sizes + [\u0027OVER\u0027]])"},{"line_number":64,"context_line":""},{"line_number":65,"context_line":"    def create_recon_nested_dict(self, top_level_key, device_list, item):"},{"line_number":66,"context_line":"        if device_list:"},{"line_number":67,"context_line":"            device_key \u003d \u0027\u0027.join(device_list)"},{"line_number":68,"context_line":"            return {top_level_key: {device_key: item}}"},{"line_number":69,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhMc%3D","line":66,"in_reply_to":"AAAAUX%2F%2Fh3g%3D","updated":"2014-02-12 16:37:31.000000000","message":"Done","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"    def audit_all_objects(self, mode\u003d\u0027once\u0027, device_dirs\u003dNone):"},{"line_number":73,"context_line":"        description \u003d \u0027\u0027"},{"line_number":74,"context_line":"        if device_dirs:"},{"line_number":75,"context_line":"            device_dir_str \u003d \u0027,\u0027.join(device_dirs)"},{"line_number":76,"context_line":"            if self.auditor_type \u003d\u003d \u0027ALL\u0027:"},{"line_number":77,"context_line":"                description \u003d _(\u0027 - parallel, %s\u0027) % device_dir_str"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fh3Q%3D","line":74,"updated":"2014-02-12 15:42:27.000000000","message":"Should the devices be sorted so the string is the same each time the device is list has the same devices?","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":71,"context_line":""},{"line_number":72,"context_line":"    def audit_all_objects(self, mode\u003d\u0027once\u0027, device_dirs\u003dNone):"},{"line_number":73,"context_line":"        description \u003d \u0027\u0027"},{"line_number":74,"context_line":"        if device_dirs:"},{"line_number":75,"context_line":"            device_dir_str \u003d \u0027,\u0027.join(device_dirs)"},{"line_number":76,"context_line":"            if self.auditor_type \u003d\u003d \u0027ALL\u0027:"},{"line_number":77,"context_line":"                description \u003d _(\u0027 - parallel, %s\u0027) % device_dir_str"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhMY%3D","line":74,"in_reply_to":"AAAAUX%2F%2Fh3Q%3D","updated":"2014-02-12 16:37:31.000000000","message":"Done","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            try:"},{"line_number":279,"context_line":"                self.run_audit(**kwargs)"},{"line_number":280,"context_line":"            except (Exception, Timeout):"},{"line_number":281,"context_line":"                self.logger.exception(_(\u0027ERROR auditing\u0027))"},{"line_number":282,"context_line":"            if kwargs[\u0027mode\u0027] \u003d\u003d \u0027forever\u0027:"},{"line_number":283,"context_line":"                self._sleep()"},{"line_number":284,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fh2Y%3D","line":281,"updated":"2014-02-12 15:42:27.000000000","message":"I\u0027d say move the sleep back to the run_forever method.","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":278,"context_line":"            try:"},{"line_number":279,"context_line":"                self.run_audit(**kwargs)"},{"line_number":280,"context_line":"            except (Exception, Timeout):"},{"line_number":281,"context_line":"                self.logger.exception(_(\u0027ERROR auditing\u0027))"},{"line_number":282,"context_line":"            if kwargs[\u0027mode\u0027] \u003d\u003d \u0027forever\u0027:"},{"line_number":283,"context_line":"                self._sleep()"},{"line_number":284,"context_line":"        else:"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhMU%3D","line":281,"in_reply_to":"AAAAUX%2F%2Fh2Y%3D","updated":"2014-02-12 16:37:31.000000000","message":"Done","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":320,"context_line":"                    kwargs[\u0027device_dirs\u0027] \u003d override_devices"},{"line_number":321,"context_line":"                    zbf_pid \u003d self.fork_child(zero_byte_fps\u003dTrue, **kwargs)"},{"line_number":322,"context_line":"                    pids.append(zbf_pid)"},{"line_number":323,"context_line":"                pids.remove(pid)"},{"line_number":324,"context_line":"            if kwargs[\u0027mode\u0027] \u003d\u003d \u0027forever\u0027:"},{"line_number":325,"context_line":"                self._sleep()"},{"line_number":326,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fh2U%3D","line":323,"updated":"2014-02-12 15:42:27.000000000","message":"I\u0027d say move the sleep back to the run_forever method.","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":320,"context_line":"                    kwargs[\u0027device_dirs\u0027] \u003d override_devices"},{"line_number":321,"context_line":"                    zbf_pid \u003d self.fork_child(zero_byte_fps\u003dTrue, **kwargs)"},{"line_number":322,"context_line":"                    pids.append(zbf_pid)"},{"line_number":323,"context_line":"                pids.remove(pid)"},{"line_number":324,"context_line":"            if kwargs[\u0027mode\u0027] \u003d\u003d \u0027forever\u0027:"},{"line_number":325,"context_line":"                self._sleep()"},{"line_number":326,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhL4%3D","line":323,"in_reply_to":"AAAAUX%2F%2Fh2U%3D","updated":"2014-02-12 16:37:31.000000000","message":"Done","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":333,"context_line":"            # only start parent"},{"line_number":334,"context_line":"            parent \u003d True"},{"line_number":335,"context_line":"        kwargs \u003d {\u0027mode\u0027: \u0027forever\u0027}"},{"line_number":336,"context_line":""},{"line_number":337,"context_line":"        while True:"},{"line_number":338,"context_line":"            self.parallel_audit_loop(parent, zbo_fps, **kwargs)"},{"line_number":339,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fh1M%3D","line":336,"updated":"2014-02-12 15:42:27.000000000","message":"It\u0027s a little strange there is no call to run_once(), but maybe that\u0027s ok.\n\nRun once gets override devices from kwargs but run_forever doesn\u0027t?  We don\u0027t want to be able specify override devices for this method?\n\nThere needs to be a try/except around the call that does the work.  Otherwise an exception or timeout will cause this script to exit.  It\u0027d be better if it just logged the exception, slept, and tried again.","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":333,"context_line":"            # only start parent"},{"line_number":334,"context_line":"            parent \u003d True"},{"line_number":335,"context_line":"        kwargs \u003d {\u0027mode\u0027: \u0027forever\u0027}"},{"line_number":336,"context_line":""},{"line_number":337,"context_line":"        while True:"},{"line_number":338,"context_line":"            self.parallel_audit_loop(parent, zbo_fps, **kwargs)"},{"line_number":339,"context_line":""}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhJM%3D","line":336,"in_reply_to":"AAAAUX%2F%2Fh1M%3D","updated":"2014-02-12 16:37:31.000000000","message":"I\u0027ve added a try/except \u0026 log around the call to parallel_audit_loop in both run_once and run_forever.  run_once is only called if the object-auditor is invoked from the command-line.  override_devices is only to be set when running in once mode, and allows you to specify a sub-set of devices to audit.  This is something that was requested by Clay.\n\nI hope that clarifies things.","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"cb10f3498a82d2a7cd825667e799b2bd5d7c7721","unresolved":false,"context_lines":[{"line_number":350,"context_line":"                                     override_devices\u003doverride_devices,"},{"line_number":351,"context_line":"                                     **kwargs)"},{"line_number":352,"context_line":"        except (Exception, Timeout):"},{"line_number":353,"context_line":"            self.logger.exception(_(\u0027ERROR auditing\u0027))"}],"source_content_type":"text/x-python","patch_set":15,"id":"AAAAXH%2F%2FnZw%3D","line":353,"updated":"2014-04-18 19:35:26.000000000","message":"I\u0027m pretty sure you backed out a change made by Paul in 968ec980 here.","commit_id":"d2a0ff3b874eff03e40bb637338e9f58302b9118"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"f5b14fef9bc43ddedda31eb22eae7ad2c7b10df9","unresolved":false,"context_lines":[{"line_number":264,"context_line":"            self.run_audit(**kwargs)"},{"line_number":265,"context_line":"            sys.exit()"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"    def parallel_audit_loop(self, parent, zbo_fps, override_devices\u003dNone,"},{"line_number":268,"context_line":"                            **kwargs):"},{"line_number":269,"context_line":"        \"\"\"Parallel audit loop\"\"\""},{"line_number":270,"context_line":"        self.clear_recon_cache(\u0027ALL\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9ad9bd40_93cf6cb0","line":267,"updated":"2014-04-30 01:54:20.000000000","message":"Was this rename really necessary?","commit_id":"16d7f9df638ecc25c518fb82aafa4f88b8121d62"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"6f1dd50cff4c7474de9684ae8371f5389d9f6bf4","unresolved":false,"context_lines":[{"line_number":264,"context_line":"            self.run_audit(**kwargs)"},{"line_number":265,"context_line":"            sys.exit()"},{"line_number":266,"context_line":""},{"line_number":267,"context_line":"    def parallel_audit_loop(self, parent, zbo_fps, override_devices\u003dNone,"},{"line_number":268,"context_line":"                            **kwargs):"},{"line_number":269,"context_line":"        \"\"\"Parallel audit loop\"\"\""},{"line_number":270,"context_line":"        self.clear_recon_cache(\u0027ALL\u0027)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9ad9bd40_febdd803","line":267,"in_reply_to":"9ad9bd40_93cf6cb0","updated":"2014-04-30 08:20:18.000000000","message":"Not really.  Reverted.","commit_id":"16d7f9df638ecc25c518fb82aafa4f88b8121d62"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"75b906c6cc27d95736b59b28972c0b69870ec67d","unresolved":false,"context_lines":[{"line_number":76,"context_line":"            if self.auditor_type \u003d\u003d \u0027ALL\u0027:"},{"line_number":77,"context_line":"                description \u003d _(\u0027 - parallel, %s\u0027) % device_dir_str"},{"line_number":78,"context_line":"            else:"},{"line_number":79,"context_line":"                description \u003d _(\u0027 - %s\u0027) % device_dir_str"},{"line_number":80,"context_line":"        self.logger.info(_(\u0027Begin object audit \"%s\" mode (%s%s)\u0027) %"},{"line_number":81,"context_line":"                        (mode, self.auditor_type, description))"},{"line_number":82,"context_line":"        begin \u003d reported \u003d time.time()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1ae5cdf2_d671d8a9","line":79,"updated":"2014-06-16 13:21:01.000000000","message":"unit test coverage for this line has been lost. maybe add this test:\nhttps://gist.github.com/alistairncoles/56bff006e32b0563abc6","commit_id":"cf990df1c83724dc8a71e6e3b46926ec369466c5"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"274eaff62f1d6d0375756a2f093418573a97e366","unresolved":false,"context_lines":[{"line_number":76,"context_line":"            if self.auditor_type \u003d\u003d \u0027ALL\u0027:"},{"line_number":77,"context_line":"                description \u003d _(\u0027 - parallel, %s\u0027) % device_dir_str"},{"line_number":78,"context_line":"            else:"},{"line_number":79,"context_line":"                description \u003d _(\u0027 - %s\u0027) % device_dir_str"},{"line_number":80,"context_line":"        self.logger.info(_(\u0027Begin object audit \"%s\" mode (%s%s)\u0027) %"},{"line_number":81,"context_line":"                        (mode, self.auditor_type, description))"},{"line_number":82,"context_line":"        begin \u003d reported \u003d time.time()"}],"source_content_type":"text/x-python","patch_set":18,"id":"1ae5cdf2_b94d189c","line":79,"in_reply_to":"1ae5cdf2_d671d8a9","updated":"2014-06-16 13:32:54.000000000","message":"Thanks Alistair.  See patch-set 19.","commit_id":"cf990df1c83724dc8a71e6e3b46926ec369466c5"}],"swift/obj/diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"b48a809d7ebe714e76d327edd654797df3b5e926","unresolved":false,"context_lines":[{"line_number":307,"context_line":""},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"def object_audit_location_generator(devices, mount_check\u003dTrue, logger\u003dNone,"},{"line_number":310,"context_line":"                                    device_dir\u003d[]):"},{"line_number":311,"context_line":"    \"\"\""},{"line_number":312,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":313,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2F4SA%3D","line":310,"updated":"2014-01-07 10:54:16.000000000","message":"using a mutable type as a default argument to a keyword in python is generally avoided:\n\nhttp://effbot.org/zone/default-values.htm","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"faa5316777281320d1bf9d4a4caad26519f0b2b0","unresolved":false,"context_lines":[{"line_number":307,"context_line":""},{"line_number":308,"context_line":""},{"line_number":309,"context_line":"def object_audit_location_generator(devices, mount_check\u003dTrue, logger\u003dNone,"},{"line_number":310,"context_line":"                                    device_dir\u003d[]):"},{"line_number":311,"context_line":"    \"\"\""},{"line_number":312,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":313,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"}],"source_content_type":"text/x-python","patch_set":2,"id":"AAAAUH%2F%2Fm3U%3D","line":310,"in_reply_to":"AAAAUH%2F%2F4SA%3D","updated":"2014-01-10 14:55:59.000000000","message":"See patch-set 3","commit_id":"8efabea1f512d4197ddccfcf6b9300f1eefb13b2"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"9a6a5ffa69d6e0599aa329767f449d356d81c4f0","unresolved":false,"context_lines":[{"line_number":349,"context_line":""},{"line_number":350,"context_line":""},{"line_number":351,"context_line":"def object_audit_location_generator(devices, mount_check\u003dTrue, logger\u003dNone,"},{"line_number":352,"context_line":"                                    device_dir\u003dNone):"},{"line_number":353,"context_line":"    \"\"\""},{"line_number":354,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":355,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2B5Eg%3D","line":352,"updated":"2014-01-21 01:39:02.000000000","message":"Tiny nitpicky thing here, but device_dir is a collection of things; could you make its name plural?","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"29e2e7cb1a1f9c28bb8ab9389ea71d77ebc7c297","unresolved":false,"context_lines":[{"line_number":349,"context_line":""},{"line_number":350,"context_line":""},{"line_number":351,"context_line":"def object_audit_location_generator(devices, mount_check\u003dTrue, logger\u003dNone,"},{"line_number":352,"context_line":"                                    device_dir\u003dNone):"},{"line_number":353,"context_line":"    \"\"\""},{"line_number":354,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":355,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2Bw8A%3D","line":352,"in_reply_to":"AAAAUH%2F%2B5Eg%3D","updated":"2014-01-22 11:41:00.000000000","message":"Done","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"9a6a5ffa69d6e0599aa329767f449d356d81c4f0","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \"\"\""},{"line_number":354,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":355,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"},{"line_number":356,"context_line":"    is set only yield AuditLocation for the objects under the entries in"},{"line_number":357,"context_line":"    device_dir. The AuditLocation only knows the path to the hash directory,"},{"line_number":358,"context_line":"    not to the .data file therein (if any). This is to avoid a double"},{"line_number":359,"context_line":"    listdir(hash_dir); the DiskFile object will always do one, so we don\u0027t."}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2B5EI%3D","line":356,"updated":"2014-01-21 01:39:02.000000000","message":"comma after \"set\"","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"29e2e7cb1a1f9c28bb8ab9389ea71d77ebc7c297","unresolved":false,"context_lines":[{"line_number":353,"context_line":"    \"\"\""},{"line_number":354,"context_line":"    Given a devices path (e.g. \"/srv/node\"), yield an AuditLocation for all"},{"line_number":355,"context_line":"    objects stored under that directory if device_dir isn\u0027t set.  If device_dir"},{"line_number":356,"context_line":"    is set only yield AuditLocation for the objects under the entries in"},{"line_number":357,"context_line":"    device_dir. The AuditLocation only knows the path to the hash directory,"},{"line_number":358,"context_line":"    not to the .data file therein (if any). This is to avoid a double"},{"line_number":359,"context_line":"    listdir(hash_dir); the DiskFile object will always do one, so we don\u0027t."}],"source_content_type":"text/x-python","patch_set":4,"id":"AAAAUH%2F%2Bw78%3D","line":356,"in_reply_to":"AAAAUH%2F%2B5EI%3D","updated":"2014-01-22 11:41:00.000000000","message":"Done","commit_id":"3634086848d41665fc9707d424db1cbfcb2d41c4"},{"author":{"_account_id":2622,"name":"Samuel Merritt","email":"spam+launchpad@andcheese.org","username":"torgomatic"},"change_message_id":"847c0dae73b2a44c5dd2b01e0d2357fc6316bb37","unresolved":false,"context_lines":[{"line_number":371,"context_line":"    if not device_dirs:"},{"line_number":372,"context_line":"        device_dirs \u003d listdir(devices)"},{"line_number":373,"context_line":"        # randomize devices in case of process restart before sweep completed"},{"line_number":374,"context_line":"        shuffle(device_dirs)"},{"line_number":375,"context_line":""},{"line_number":376,"context_line":"    for device in device_dirs:"},{"line_number":377,"context_line":"        if mount_check and not \\"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2BhJ8%3D","line":374,"updated":"2014-01-24 20:27:56.000000000","message":"I think you need to intersect the passed-in device dirs with the result of this listdir.\n\nOtherwise, what can happen is:\n\n* you call this with device_dirs\u003d(\u0027sdf\u0027, \u0027sdg\u0027, \u0027bogus\u0027, \u0027sdh\u0027)\n\n* the listdir(datadir_path) below fails with OSError/ENOTDIR\n\n* the exception bubbles up to the auditor and kills it\n\nIf you have mount_check\u003dTrue, the same thing will happen, but ismount() is the one that blows up with OSError/ENOTDIR, so you don\u0027t make it to listdir().","commit_id":"dbd397a0dff35ca473cd01653acdf00cf704bc8d"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"0431686cc1a5d8cddb2f71df2cb586cf691d0033","unresolved":false,"context_lines":[{"line_number":371,"context_line":"    if not device_dirs:"},{"line_number":372,"context_line":"        device_dirs \u003d listdir(devices)"},{"line_number":373,"context_line":"        # randomize devices in case of process restart before sweep completed"},{"line_number":374,"context_line":"        shuffle(device_dirs)"},{"line_number":375,"context_line":""},{"line_number":376,"context_line":"    for device in device_dirs:"},{"line_number":377,"context_line":"        if mount_check and not \\"}],"source_content_type":"text/x-python","patch_set":5,"id":"AAAAUH%2F%2BYUU%3D","line":374,"in_reply_to":"AAAAUH%2F%2BhJ8%3D","updated":"2014-01-27 16:18:33.000000000","message":"You\u0027re absolutely correct Sam - good catch.  I\u0027ve added code to remove bogus and duplicate entries in the device_dirs list in patch set 6.  I\u0027ve added code both in ObjectAuditor.run_once (where a user can set the list of devices at the command line) and again in object_audit_location_generator (in case at some point in the future device_dirs is set somewhere else outside the code that I\u0027m submitting).  The code added in ObjectAuditor.run_once might seem superfluous, however logging uses the device_dirs list before it gets to the object_audit_location_generator and it would be good to fix-up device_dirs for logging.","commit_id":"dbd397a0dff35ca473cd01653acdf00cf704bc8d"},{"author":{"_account_id":1009,"name":"Greg Lange","email":"greglange@gmail.com","username":"greglange"},"change_message_id":"06fb1097bd1444d821d0ac4fa5ff03060b3a557d","unresolved":false,"context_lines":[{"line_number":376,"context_line":"        # remove bogus devices and duplicates from device_dirs"},{"line_number":377,"context_line":"        device_dirs \u003d list("},{"line_number":378,"context_line":"            set(listdir(devices)).intersection(set(device_dirs)))"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"    for device in device_dirs:"},{"line_number":381,"context_line":"        if mount_check and not \\"},{"line_number":382,"context_line":"                ismount(os.path.join(devices, device)):"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2Fhy4%3D","line":379,"updated":"2014-02-12 15:42:27.000000000","message":"Move shuffle(device_dirs) down here so that the devices get shuffled either way?","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"7f18a921b71106d284347eb2e49333d20fd148c2","unresolved":false,"context_lines":[{"line_number":376,"context_line":"        # remove bogus devices and duplicates from device_dirs"},{"line_number":377,"context_line":"        device_dirs \u003d list("},{"line_number":378,"context_line":"            set(listdir(devices)).intersection(set(device_dirs)))"},{"line_number":379,"context_line":""},{"line_number":380,"context_line":"    for device in device_dirs:"},{"line_number":381,"context_line":"        if mount_check and not \\"},{"line_number":382,"context_line":"                ismount(os.path.join(devices, device)):"}],"source_content_type":"text/x-python","patch_set":11,"id":"AAAAUX%2F%2FhMg%3D","line":379,"in_reply_to":"AAAAUX%2F%2Fhy4%3D","updated":"2014-02-12 16:37:31.000000000","message":"Done","commit_id":"94a9cf16b1a4994b61a5fa5194e758b633a8b880"},{"author":{"_account_id":597,"name":"Pete Zaitcev","email":"zaitcev@kotori.zaitcev.us","username":"zaitcev"},"change_message_id":"f5b14fef9bc43ddedda31eb22eae7ad2c7b10df9","unresolved":false,"context_lines":[{"line_number":39,"context_line":"import logging"},{"line_number":40,"context_line":"import traceback"},{"line_number":41,"context_line":"from random import shuffle"},{"line_number":42,"context_line":"from os.path import basename, dirname, exists, getmtime, join"},{"line_number":43,"context_line":"from tempfile import mkstemp"},{"line_number":44,"context_line":"from contextlib import contextmanager"},{"line_number":45,"context_line":"from collections import defaultdict"}],"source_content_type":"text/x-python","patch_set":16,"id":"9ad9bd40_13e43c3a","line":42,"updated":"2014-04-30 01:54:20.000000000","message":"I don\u0027t see any value in this change.","commit_id":"16d7f9df638ecc25c518fb82aafa4f88b8121d62"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"6f1dd50cff4c7474de9684ae8371f5389d9f6bf4","unresolved":false,"context_lines":[{"line_number":39,"context_line":"import logging"},{"line_number":40,"context_line":"import traceback"},{"line_number":41,"context_line":"from random import shuffle"},{"line_number":42,"context_line":"from os.path import basename, dirname, exists, getmtime, join"},{"line_number":43,"context_line":"from tempfile import mkstemp"},{"line_number":44,"context_line":"from contextlib import contextmanager"},{"line_number":45,"context_line":"from collections import defaultdict"}],"source_content_type":"text/x-python","patch_set":16,"id":"9ad9bd40_bec35089","line":42,"in_reply_to":"9ad9bd40_13e43c3a","updated":"2014-04-30 08:20:18.000000000","message":"Reverted","commit_id":"16d7f9df638ecc25c518fb82aafa4f88b8121d62"}],"test/unit/obj/test_auditor.py":[{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"930b9abf6ee1281c6e79ad2270a3054f96ed3ae1","unresolved":false,"context_lines":[{"line_number":421,"context_line":"            self.assertEquals(mocker.fork_called, 2)"},{"line_number":422,"context_line":"            self.assertEquals(mocker.check_kwargs[\u0027zero_byte_fps\u0027], 89)"},{"line_number":423,"context_line":""},{"line_number":424,"context_line":"            my_auditor.concurrency \u003d len(os.listdir(self.devices)) + 1"},{"line_number":425,"context_line":"            mocker.fork_called \u003d 0"},{"line_number":426,"context_line":"            self.assertRaises(OSError, my_auditor.run_forever)"},{"line_number":427,"context_line":"            self.assertEquals(mocker.fork_called, my_auditor.concurrency)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F9A7c%3D","line":424,"updated":"2013-12-17 17:48:45.000000000","message":"In the real world this is most likely bigger than one, but definetely lower than the amount of devices. I think it would be good to get this covered as well to ensure that the replication works as expected","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"},{"author":{"_account_id":1005,"name":"Eamonn O\u0027Toole","email":"eamonn.otoole@hp.com","username":"eamonn-otoole"},"change_message_id":"f967e65ea97b2e5a6f45f4186d10f3b301b67e8a","unresolved":false,"context_lines":[{"line_number":421,"context_line":"            self.assertEquals(mocker.fork_called, 2)"},{"line_number":422,"context_line":"            self.assertEquals(mocker.check_kwargs[\u0027zero_byte_fps\u0027], 89)"},{"line_number":423,"context_line":""},{"line_number":424,"context_line":"            my_auditor.concurrency \u003d len(os.listdir(self.devices)) + 1"},{"line_number":425,"context_line":"            mocker.fork_called \u003d 0"},{"line_number":426,"context_line":"            self.assertRaises(OSError, my_auditor.run_forever)"},{"line_number":427,"context_line":"            self.assertEquals(mocker.fork_called, my_auditor.concurrency)"}],"source_content_type":"text/x-python","patch_set":1,"id":"AAAATn%2F8yCg%3D","line":424,"in_reply_to":"AAAATn%2F9A7c%3D","updated":"2013-12-20 11:39:41.000000000","message":"Look at the updated patch","commit_id":"2f5208c2d40f16ff29f1c5e221def04108976876"}]}