)]}'
{"doc/source/deployment_guide.rst":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4de6edc0df4d72b7c0253932de6cfff424d279d","unresolved":true,"context_lines":[{"line_number":497,"context_line":"*************************"},{"line_number":498,"context_line":""},{"line_number":499,"context_line":"When a container gets :ref:`sharded\u003csharding_doc\u003e` the root container (the original) will still be the"},{"line_number":500,"context_line":"primary entry point to container GET requests, as it provides the list of shards."},{"line_number":501,"context_line":"To take load off the root container swift by default caches the list of shards returned."},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"As the number of shards for a root container grows to more then 3k the memcache max size"}],"source_content_type":"text/x-rst","patch_set":1,"id":"ea280eae_f27d8e8b","line":500,"updated":"2021-01-07 09:37:40.000000000","message":"nit: We also cache shard ranges for object updates, so perhaps simplify to just point out that when a container is sharded the proxy caches shard range info in memcache.","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4de6edc0df4d72b7c0253932de6cfff424d279d","unresolved":true,"context_lines":[{"line_number":498,"context_line":""},{"line_number":499,"context_line":"When a container gets :ref:`sharded\u003csharding_doc\u003e` the root container (the original) will still be the"},{"line_number":500,"context_line":"primary entry point to container GET requests, as it provides the list of shards."},{"line_number":501,"context_line":"To take load off the root container swift by default caches the list of shards returned."},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"As the number of shards for a root container grows to more then 3k the memcache max size"},{"line_number":504,"context_line":"of 1MB can be reached. Care is needed to keep an eye on the current value or resize it to"}],"source_content_type":"text/x-rst","patch_set":1,"id":"02cd7bf0_7c17f71a","line":501,"updated":"2021-01-07 09:37:40.000000000","message":"I think we usually capitalize Swift in docs","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e4de6edc0df4d72b7c0253932de6cfff424d279d","unresolved":true,"context_lines":[{"line_number":500,"context_line":"primary entry point to container GET requests, as it provides the list of shards."},{"line_number":501,"context_line":"To take load off the root container swift by default caches the list of shards returned."},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"As the number of shards for a root container grows to more then 3k the memcache max size"},{"line_number":504,"context_line":"of 1MB can be reached. Care is needed to keep an eye on the current value or resize it to"},{"line_number":505,"context_line":"something like 5MB."},{"line_number":506,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"8bcd9ee1_e789926e","line":503,"updated":"2021-01-07 09:37:40.000000000","message":"s/then/than/\n\nalso, s/max size/default max value size/","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2bd08131ce95e0dd2a41bcf1caa2eed8584837e8","unresolved":true,"context_lines":[{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"-----------"},{"line_number":519,"context_line":"System Time"}],"source_content_type":"text/x-rst","patch_set":1,"id":"e8410387_a9e96b19","line":516,"updated":"2021-01-07 06:17:17.000000000","message":"There is a chance that running the `stats sizes` command might loc memcache, or at least an old oracle site seemed to allude to it. Another option would be to fire off a statsd metric of the shard listing cache size as it goes into memcache so ops can capture the top/max sizes and monitor this max limit. (I assume that\u0027s possible in graphana).","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"88ce9b5e5899e2e9c8f6c12e0a24c0d82f09ea35","unresolved":true,"context_lines":[{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"-----------"},{"line_number":519,"context_line":"System Time"}],"source_content_type":"text/x-rst","patch_set":1,"id":"5beecfaf_356d29c3","line":516,"in_reply_to":"bd17694c_511c7340","updated":"2021-01-08 05:37:35.000000000","message":"Not yet, but we can! https://review.opendev.org/c/openstack/swift/+/769868\n\nThat\u0027s still going to mainly be useful to put out the fire, though -- I\u0027d prefer to be able to just avoid it 😉\n\nI like the statsd metric idea, though we might want a configurable threshold to reduce how many stats are emitted. For example, emit *just* the sizes that are over 500kB -- and once you see that you\u0027re inching dangerously close to 1MB, bump up the memcache config to 5MB and the threshold to 4MB.","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"85a611343874f5722465a007a7146d4b2cb86154","unresolved":true,"context_lines":[{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"-----------"},{"line_number":519,"context_line":"System Time"}],"source_content_type":"text/x-rst","patch_set":1,"id":"eadf21e9_0fb70b39","line":516,"in_reply_to":"e8410387_a9e96b19","updated":"2021-01-07 11:56:48.000000000","message":"I discovered that memcache returns \u0027SERVER_ERROR object too large for cache\u0027 when the set fails \n\nhere https://github.com/openstack/swift/blob/5320ecbaf2c0e77842ab1ee3eb8106948dc06704/swift/common/memcached.py#L302","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"fa67c064a5882b8143d626b249613b7c47e6fcf7","unresolved":true,"context_lines":[{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""},{"line_number":518,"context_line":"-----------"},{"line_number":519,"context_line":"System Time"}],"source_content_type":"text/x-rst","patch_set":1,"id":"bd17694c_511c7340","line":516,"in_reply_to":"eadf21e9_0fb70b39","updated":"2021-01-08 04:25:55.000000000","message":"And does that end up getting logged? If so I\u0027ll point it out in doc.","commit_id":"23261b1da5f6ade02e37ec4ab5fd1d9445b8e3b8"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"88ce9b5e5899e2e9c8f6c12e0a24c0d82f09ea35","unresolved":true,"context_lines":[{"line_number":500,"context_line":"primary entry point to many container requests, as it provides the list of shards."},{"line_number":501,"context_line":"To take load off the root container Swift by default caches the list of shards returned."},{"line_number":502,"context_line":""},{"line_number":503,"context_line":"As the number of shards for a root container grows to more than 3k the memcache default max"},{"line_number":504,"context_line":"size of 1MB can be reached. Care is needed to keep an eye on the current value or resize it"},{"line_number":505,"context_line":"to something like 5MB."},{"line_number":506,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"e85d14b6_0b09b4b0","line":503,"range":{"start_line":503,"start_character":64,"end_line":503,"end_character":66},"updated":"2021-01-08 05:37:35.000000000","message":"This is assuming our experience is typical, yeah? But I think a cluster could start bumping into this with \"just\" hundreds of shard for a single container...","commit_id":"8df3d1505a4e034983c609cb6f8f5e9a0fc945f2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"88adb637d067504cd82f8a1e573dc489a4534434","unresolved":true,"context_lines":[{"line_number":502,"context_line":""},{"line_number":503,"context_line":"As the number of shards for a root container grows to more than 3k the memcache default max"},{"line_number":504,"context_line":"size of 1MB can be reached. Care is needed to keep an eye on the current value or resize it"},{"line_number":505,"context_line":"to something like 5MB."},{"line_number":506,"context_line":""},{"line_number":507,"context_line":"Memcache has a ``stats sizes`` option that can point out the current size usage. As this"},{"line_number":508,"context_line":"reaches the current max an increase might be in order::"}],"source_content_type":"text/x-rst","patch_set":2,"id":"fac79c30_f6f22ab3","line":505,"updated":"2021-01-27 16:05:34.000000000","message":"really this last sentence here is the most useful thing - \"increase your memcache size\"","commit_id":"8df3d1505a4e034983c609cb6f8f5e9a0fc945f2"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"88ce9b5e5899e2e9c8f6c12e0a24c0d82f09ea35","unresolved":true,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"  # telnet \u003cmemcache server\u003e 11211"},{"line_number":511,"context_line":"  \u003e stats sizes"},{"line_number":512,"context_line":"  STAT 160 2"},{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"f63c8b90_85b17beb","line":514,"range":{"start_line":512,"start_character":0,"end_line":514,"end_character":12},"updated":"2021-01-08 05:37:35.000000000","message":"What does this output mean? There are two keys 160 bytes long, one key 448 bytes long, and one key 576 bytes long?","commit_id":"8df3d1505a4e034983c609cb6f8f5e9a0fc945f2"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"88adb637d067504cd82f8a1e573dc489a4534434","unresolved":true,"context_lines":[{"line_number":509,"context_line":""},{"line_number":510,"context_line":"  # telnet \u003cmemcache server\u003e 11211"},{"line_number":511,"context_line":"  \u003e stats sizes"},{"line_number":512,"context_line":"  STAT 160 2"},{"line_number":513,"context_line":"  STAT 448 1"},{"line_number":514,"context_line":"  STAT 576 1"},{"line_number":515,"context_line":"  END"},{"line_number":516,"context_line":""},{"line_number":517,"context_line":""}],"source_content_type":"text/x-rst","patch_set":2,"id":"5b6ccc9d_e4e48aab","line":514,"range":{"start_line":512,"start_character":0,"end_line":514,"end_character":12},"in_reply_to":"f63c8b90_85b17beb","updated":"2021-01-27 16:05:34.000000000","message":"yeah, i\u0027m not sure what an operator would do with this information directly, but I don\u0027t mind throwing it out there to point someone in the right direction","commit_id":"8df3d1505a4e034983c609cb6f8f5e9a0fc945f2"}]}