)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":7,"context_line":"Add doc entry to check partition count"},{"line_number":8,"context_line":""},{"line_number":9,"context_line":"An high or increasing partition count due to storing handoffs can have"},{"line_number":10,"context_line":"some severe sideeffects, and replication might never be able to catch"},{"line_number":11,"context_line":"up. This patch adds a note to the admin_guide how to check this."},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Change-Id: Ib4e161d68f1a82236dbf5fac13ef9a13ac4bbf18"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":1,"id":"dada55a8_704dca98","line":10,"updated":"2016-07-25 14:39:45.000000000","message":"nit s/sideeffects/side-effects/","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"}],"doc/source/admin_guide.rst":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":622,"context_line":"happens, it is recommended to disable write_affinity and simply let"},{"line_number":623,"context_line":"object PUTs traverse the WAN link, as that will naturally limit the"},{"line_number":624,"context_line":"object growth rate to what your WAN link can handle."},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"You can check if handoff partitions are piling up on a server by"},{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_b0b9027c","line":625,"updated":"2016-07-25 14:39:45.000000000","message":"nit: I wonder if it is worth having a sub-heading for this new material?","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":622,"context_line":"happens, it is recommended to disable write_affinity and simply let"},{"line_number":623,"context_line":"object PUTs traverse the WAN link, as that will naturally limit the"},{"line_number":624,"context_line":"object growth rate to what your WAN link can handle."},{"line_number":625,"context_line":""},{"line_number":626,"context_line":"You can check if handoff partitions are piling up on a server by"},{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_a10532c3","line":625,"in_reply_to":"dada55a8_b0b9027c","updated":"2016-07-25 16:01:56.000000000","message":"Good idea, will add one","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":625,"context_line":""},{"line_number":626,"context_line":"You can check if handoff partitions are piling up on a server by"},{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"},{"line_number":629,"context_line":"assigned to a server using the ``dispersion`` command from"},{"line_number":630,"context_line":"``swift-ring-builder``::"},{"line_number":631,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_6d5052d5","line":628,"updated":"2016-07-25 14:39:45.000000000","message":"s/is/are/","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":625,"context_line":""},{"line_number":626,"context_line":"You can check if handoff partitions are piling up on a server by"},{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"},{"line_number":629,"context_line":"assigned to a server using the ``dispersion`` command from"},{"line_number":630,"context_line":"``swift-ring-builder``::"},{"line_number":631,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_b601fa6c","line":628,"in_reply_to":"dada55a8_6d5052d5","updated":"2016-07-25 16:01:56.000000000","message":"Done","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"},{"line_number":629,"context_line":"assigned to a server using the ``dispersion`` command from"},{"line_number":630,"context_line":"``swift-ring-builder``::"},{"line_number":631,"context_line":""},{"line_number":632,"context_line":"    swift-ring-builder /etc/swift/object.builder dispersion -v r1z1-172.16.10.5"},{"line_number":633,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_6d2a1292","line":630,"updated":"2016-07-25 14:39:45.000000000","message":"Probably worth being clear that the device name here is an example.\n\nAre you assuming that *any* server in region 1 could be checked because objects will be uniformly distributed? If so, worth pointing that out.","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":627,"context_line":"comparing the expected number of partitions with the actual number on"},{"line_number":628,"context_line":"your disks. First get the number of partitions that is currently"},{"line_number":629,"context_line":"assigned to a server using the ``dispersion`` command from"},{"line_number":630,"context_line":"``swift-ring-builder``::"},{"line_number":631,"context_line":""},{"line_number":632,"context_line":"    swift-ring-builder /etc/swift/object.builder dispersion -v r1z1-172.16.10.5"},{"line_number":633,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_76cae21b","line":630,"in_reply_to":"dada55a8_6d2a1292","updated":"2016-07-25 16:01:56.000000000","message":"That would be only the case if there are more servers than replicas (which is likely, however).","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":629,"context_line":"assigned to a server using the ``dispersion`` command from"},{"line_number":630,"context_line":"``swift-ring-builder``::"},{"line_number":631,"context_line":""},{"line_number":632,"context_line":"    swift-ring-builder /etc/swift/object.builder dispersion -v r1z1-172.16.10.5"},{"line_number":633,"context_line":""},{"line_number":634,"context_line":"Now count the number of object partitions on that server::"},{"line_number":635,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_8dc78511","line":632,"updated":"2016-07-25 14:39:45.000000000","message":"nit: show example output to highlight the partition count value.\n\npossibly unnecessary for admin guide if we are assuming familiarity with the swift-ring-builder tool?","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":633,"context_line":""},{"line_number":634,"context_line":"Now count the number of object partitions on that server::"},{"line_number":635,"context_line":""},{"line_number":636,"context_line":"    find -L /srv/node/ -maxdepth 3 -type d -wholename \"*objects/*\" | wc -l"},{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_4d910d99","line":636,"updated":"2016-07-25 14:39:45.000000000","message":"I think some mention should be made of policy and that \"objects\" may need to be \"objects-1\" etc\n\nAlso /srv/node is configurable.\n\nWe don\u0027t seem to have a tool like swift-get-nodes that could be run on an object-server to get the actual path to an object given object path and policy.","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":20508,"name":"Mohit Motiani","email":"mohit.motiani@intel.com","username":"mohit.motiani"},"change_message_id":"60bc2f022ae767ba61617df1f016ea39e17970eb","unresolved":false,"context_lines":[{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"},{"line_number":640,"context_line":"swift-ring-builder command. The amount of additional (handoff)"},{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"7aa08908_383b6d9e","line":640,"range":{"start_line":640,"start_character":32,"end_line":640,"end_character":38},"updated":"2016-06-09 18:49:14.000000000","message":"Nit: I am not sure if amount is the correct word at this place. Number should be used in place of amount.\ns/amount/number\nSomeone else should confirm.","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":636,"context_line":"    find -L /srv/node/ -maxdepth 3 -type d -wholename \"*objects/*\" | wc -l"},{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"},{"line_number":640,"context_line":"swift-ring-builder command. The amount of additional (handoff)"},{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_0d039543","line":640,"range":{"start_line":639,"start_character":41,"end_line":640,"end_character":26},"updated":"2016-07-25 14:39:45.000000000","message":"nit: insert dispersion, so :\n\n  reported by the\nswift-ring-builder dispersion command","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"},{"line_number":640,"context_line":"swift-ring-builder command. The amount of additional (handoff)"},{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_ed7e0269","line":640,"range":{"start_line":640,"start_character":32,"end_line":640,"end_character":38},"in_reply_to":"7aa08908_383b6d9e","updated":"2016-07-25 14:39:45.000000000","message":"number might be better","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"},{"line_number":640,"context_line":"swift-ring-builder command. The amount of additional (handoff)"},{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_c1eb0677","line":640,"in_reply_to":"dada55a8_0d039543","updated":"2016-07-25 16:01:56.000000000","message":"Done","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":637,"context_line":""},{"line_number":638,"context_line":"With write_affinity enabled it is expected to have a higher number of"},{"line_number":639,"context_line":"partitions on disk compared to the value reported by the"},{"line_number":640,"context_line":"swift-ring-builder command. The amount of additional (handoff)"},{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_e1e8ca7e","line":640,"in_reply_to":"dada55a8_ed7e0269","updated":"2016-07-25 16:01:56.000000000","message":"Done","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":641,"context_line":"partitions in region r1 and r2 depends on your cluster size, the amount"},{"line_number":642,"context_line":"of incoming data as well as the replication speed."},{"line_number":643,"context_line":""},{"line_number":644,"context_line":"Let\u0027s assume you have 6 nodes in 3 regions, and write_affinity is"},{"line_number":645,"context_line":"configured to write to region r1 and r2 first. swift-ring-builder"},{"line_number":646,"context_line":"reported that each node should store ~1365 partitions and there are 3"},{"line_number":647,"context_line":"replicas."}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_30e35235","line":644,"updated":"2016-07-25 14:39:45.000000000","message":"Line 509 says that this section assumes two regions but here we move to 3. is there a useful example with only two regions?","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":644,"context_line":"Let\u0027s assume you have 6 nodes in 3 regions, and write_affinity is"},{"line_number":645,"context_line":"configured to write to region r1 and r2 first. swift-ring-builder"},{"line_number":646,"context_line":"reported that each node should store ~1365 partitions and there are 3"},{"line_number":647,"context_line":"replicas."},{"line_number":648,"context_line":""},{"line_number":649,"context_line":" Expected partitions for 2 nodes in region r3:            2 x 1365 \u003d 2730"},{"line_number":650,"context_line":" Handoffs stored across 4 nodes in region r1 and r2:      2730 / 4 ~ 683"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_f0c9ba7a","line":647,"updated":"2016-07-25 14:39:45.000000000","message":"I got confused here - what is the partition power of the ring? 6 nodes @ 1365 partitions per node -\u003e 8192 partitions? but for 3 replicas??","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":646,"context_line":"reported that each node should store ~1365 partitions and there are 3"},{"line_number":647,"context_line":"replicas."},{"line_number":648,"context_line":""},{"line_number":649,"context_line":" Expected partitions for 2 nodes in region r3:            2 x 1365 \u003d 2730"},{"line_number":650,"context_line":" Handoffs stored across 4 nodes in region r1 and r2:      2730 / 4 ~ 683"},{"line_number":651,"context_line":" Maximum number of partitions in region r1 and r2:      1365 + 683 \u003d 2048"},{"line_number":652,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_9095dee5","line":649,"updated":"2016-07-25 14:39:45.000000000","message":"These three lines are not being rendered as a list in html.","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"1bdb36f94a14c889acb45da1a2f7df66e6757feb","unresolved":false,"context_lines":[{"line_number":646,"context_line":"reported that each node should store ~1365 partitions and there are 3"},{"line_number":647,"context_line":"replicas."},{"line_number":648,"context_line":""},{"line_number":649,"context_line":" Expected partitions for 2 nodes in region r3:            2 x 1365 \u003d 2730"},{"line_number":650,"context_line":" Handoffs stored across 4 nodes in region r1 and r2:      2730 / 4 ~ 683"},{"line_number":651,"context_line":" Maximum number of partitions in region r1 and r2:      1365 + 683 \u003d 2048"},{"line_number":652,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_5c353dd8","line":649,"in_reply_to":"dada55a8_9095dee5","updated":"2016-07-25 16:01:56.000000000","message":"Done","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":648,"context_line":""},{"line_number":649,"context_line":" Expected partitions for 2 nodes in region r3:            2 x 1365 \u003d 2730"},{"line_number":650,"context_line":" Handoffs stored across 4 nodes in region r1 and r2:      2730 / 4 ~ 683"},{"line_number":651,"context_line":" Maximum number of partitions in region r1 and r2:      1365 + 683 \u003d 2048"},{"line_number":652,"context_line":""},{"line_number":653,"context_line":"Worst case is that replication never finishes and region r1 and r2 store"},{"line_number":654,"context_line":"all handoff partitions. In that case you will see ~ 2048 partitions per"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_9012feb2","line":651,"updated":"2016-07-25 14:39:45.000000000","message":"these numbers are aggregated for each region but above we used the dispersion command to get expected numbers per device - can we assume uniformity and continue to work in terms of partitions per device?","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":20508,"name":"Mohit Motiani","email":"mohit.motiani@intel.com","username":"mohit.motiani"},"change_message_id":"60bc2f022ae767ba61617df1f016ea39e17970eb","unresolved":false,"context_lines":[{"line_number":656,"context_line":"between 1365 and 2048 partitions (preferably on the lower side)."},{"line_number":657,"context_line":""},{"line_number":658,"context_line":"If the number is always on the upper end or increasing you should check"},{"line_number":659,"context_line":"your replication speed and maybe even disable write_affinity."},{"line_number":660,"context_line":""},{"line_number":661,"context_line":""},{"line_number":662,"context_line":"--------------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"7aa08908_d894e945","line":659,"range":{"start_line":659,"start_character":27,"end_line":659,"end_character":32},"updated":"2016-06-09 18:49:14.000000000","message":"Nit: s/maybe/ may be","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"b39423c95c651490f1f2993487c76d67bb347a1c","unresolved":false,"context_lines":[{"line_number":656,"context_line":"between 1365 and 2048 partitions (preferably on the lower side)."},{"line_number":657,"context_line":""},{"line_number":658,"context_line":"If the number is always on the upper end or increasing you should check"},{"line_number":659,"context_line":"your replication speed and maybe even disable write_affinity."},{"line_number":660,"context_line":""},{"line_number":661,"context_line":""},{"line_number":662,"context_line":"--------------------------------"}],"source_content_type":"text/x-rst","patch_set":1,"id":"dada55a8_eda72209","line":659,"range":{"start_line":659,"start_character":27,"end_line":659,"end_character":32},"in_reply_to":"7aa08908_d894e945","updated":"2016-07-25 14:39:45.000000000","message":"IMO \"maybe\" is correct vs \"may be\", or use \"possibly\" or \"perhaps\"","commit_id":"62b38dc17dcbdd7a32b6d7427e74c4ae003a8974"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"55dd22f0ad99a1e59da9b20ea7749532ccf41c2d","unresolved":false,"context_lines":[{"line_number":672,"context_line":"of incoming data as well as the replication speed."},{"line_number":673,"context_line":""},{"line_number":674,"context_line":"Let\u0027s use the example from above with 6 nodes in 2 regions, and write_affinity"},{"line_number":675,"context_line":"configured to write to region r1 first. swift-ring-builder reported that"},{"line_number":676,"context_line":"each node should store 4096 partitions::"},{"line_number":677,"context_line":""},{"line_number":678,"context_line":" Expected partitions for region r2:                                      8192"}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_e7def16b","line":675,"updated":"2016-07-26 09:48:26.000000000","message":"for consistency, use ``swift-ring-builder`` i.e. wrap with back ticks","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"d1cead36ad2491a1b17a33445e93d89e56773b99","unresolved":false,"context_lines":[{"line_number":672,"context_line":"of incoming data as well as the replication speed."},{"line_number":673,"context_line":""},{"line_number":674,"context_line":"Let\u0027s use the example from above with 6 nodes in 2 regions, and write_affinity"},{"line_number":675,"context_line":"configured to write to region r1 first. swift-ring-builder reported that"},{"line_number":676,"context_line":"each node should store 4096 partitions::"},{"line_number":677,"context_line":""},{"line_number":678,"context_line":" Expected partitions for region r2:                                      8192"}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_0ad0691e","line":675,"in_reply_to":"dada55a8_e7def16b","updated":"2016-07-26 12:36:18.000000000","message":"Done","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"55dd22f0ad99a1e59da9b20ea7749532ccf41c2d","unresolved":false,"context_lines":[{"line_number":679,"context_line":" Handoffs stored across 4 nodes in region r1:                 8192 / 4 \u003d 2048"},{"line_number":680,"context_line":" Maximum number of partitions on each server in region r1: 2048 + 4096 \u003d 6144"},{"line_number":681,"context_line":""},{"line_number":682,"context_line":"Worst case is that replication never finishes and region r1 stores"},{"line_number":683,"context_line":"all handoff partitions. In that case you will see ~ 6144 partitions per"},{"line_number":684,"context_line":"server in region r1. Your actual number should be lower and"},{"line_number":685,"context_line":"between 4096 and 6144 partitions (preferably on the lower side)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_a72238a1","line":682,"updated":"2016-07-26 09:48:26.000000000","message":"Maybe I am being pedantic, but replication (the process) never does \"finish\". However, I\u0027m not sure how best to describe this worst case. To maintain a constant 2048 handoff partitions on  a node a new replica would need to land in a handoff as it is being replicated, so it is never deleted. More realistically you\u0027d see a handoff replicated, deleted, and then re-populated soon after by a new object.\n\nPerhaps:\n\n  Worst case is that handoff partitions in region 1 are populated with new object replicas faster than replication is able to move them to region 2.","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"d1cead36ad2491a1b17a33445e93d89e56773b99","unresolved":false,"context_lines":[{"line_number":679,"context_line":" Handoffs stored across 4 nodes in region r1:                 8192 / 4 \u003d 2048"},{"line_number":680,"context_line":" Maximum number of partitions on each server in region r1: 2048 + 4096 \u003d 6144"},{"line_number":681,"context_line":""},{"line_number":682,"context_line":"Worst case is that replication never finishes and region r1 stores"},{"line_number":683,"context_line":"all handoff partitions. In that case you will see ~ 6144 partitions per"},{"line_number":684,"context_line":"server in region r1. Your actual number should be lower and"},{"line_number":685,"context_line":"between 4096 and 6144 partitions (preferably on the lower side)."}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_6aa765bd","line":682,"in_reply_to":"dada55a8_a72238a1","updated":"2016-07-26 12:36:18.000000000","message":"You\u0027re right, the process doesn\u0027t finish. One replication pass finishes, I should have made that more clear. OTOH I prefer your suggestion, I\u0027ll use that. Thx!","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"55dd22f0ad99a1e59da9b20ea7749532ccf41c2d","unresolved":false,"context_lines":[{"line_number":685,"context_line":"between 4096 and 6144 partitions (preferably on the lower side)."},{"line_number":686,"context_line":""},{"line_number":687,"context_line":"Now count the number of object partitions on a given server in region 1,"},{"line_number":688,"context_line":"for example on r1z1-172.16.10.1.  Note that the pathnames might be"},{"line_number":689,"context_line":"different; `/srv/node/` is the default mount location, and `objects`"},{"line_number":690,"context_line":"applies only to storage policy 0 (storage policy 1 would used"},{"line_number":691,"context_line":"`objects-1` and so on)::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_c7f00c00","line":688,"range":{"start_line":688,"start_character":14,"end_line":688,"end_character":31},"updated":"2016-07-26 09:48:26.000000000","message":"should this be just 172.16.10.1 if referring to the server rather than the device name?","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":6968,"name":"Christian Schwede","email":"cschwede@nvidia.com","username":"cschwede"},"change_message_id":"d1cead36ad2491a1b17a33445e93d89e56773b99","unresolved":false,"context_lines":[{"line_number":685,"context_line":"between 4096 and 6144 partitions (preferably on the lower side)."},{"line_number":686,"context_line":""},{"line_number":687,"context_line":"Now count the number of object partitions on a given server in region 1,"},{"line_number":688,"context_line":"for example on r1z1-172.16.10.1.  Note that the pathnames might be"},{"line_number":689,"context_line":"different; `/srv/node/` is the default mount location, and `objects`"},{"line_number":690,"context_line":"applies only to storage policy 0 (storage policy 1 would used"},{"line_number":691,"context_line":"`objects-1` and so on)::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_caa8118f","line":688,"in_reply_to":"dada55a8_c7f00c00","updated":"2016-07-26 12:36:18.000000000","message":"Done","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"55dd22f0ad99a1e59da9b20ea7749532ccf41c2d","unresolved":false,"context_lines":[{"line_number":694,"context_line":""},{"line_number":695,"context_line":"If this number is always on the upper end of the expected partition"},{"line_number":696,"context_line":"number range (4096 to 6144) or increasing you should check your"},{"line_number":697,"context_line":"replication speed and maybe even disable write_affinity."},{"line_number":698,"context_line":""},{"line_number":699,"context_line":""},{"line_number":700,"context_line":"--------------------------------"}],"source_content_type":"text/x-rst","patch_set":2,"id":"dada55a8_67a201b0","line":697,"updated":"2016-07-26 09:48:26.000000000","message":"Some suggestion of how to check replication speed/progress would be useful. Perhaps a forwards link to the next section on telemetry and an example recon command?","commit_id":"91588c29394a5ffdf73a16019773fe025f2dbd00"}]}