前言:
此话题说来话长,老大让写多个查询接口,尽量考虑索引使用,因为数据非常多,不使用索引查询很慢,于是乎,我写了一个通用查询接口,指定索引生成query对象。根据mongo权威指南的说法 建立复合索引 A_1_B_1_C_1 此时可用索引为 A-B-C |A-B| A 三种, 跟DBA说了讨论了这个问题(PS:我也不想讨论但是已经写完了真的懒得改了),老大又说不能使用通用的DB层查询接口,索引也要为业务服务,于是乎我们3个人开始验证索引顺序对查询的影响,半小时被打N次脸,难道mongo权威指南是盗版书? DBA 测试 A-B 和 B-A 都是用复合索引 A_1_B_1_C_1 且实际扫描文档数 和 返回文档树一致 2者相同。这就有点不科学了。但事实就摆在那里。。。一定有什么误会于是回家做实验了。
结论(PS 正文太长放结论先):
假设存在索引 A_1_B_1_C_1
AB与BA 都可以正常使用索引效率相同
AC 可使用索引效率较低
BC 不可使用索引 进入 COLLSCAN stage
AXBC 可使用索引与ABCX效率相同
A:{$exists:true}BC 可使用索引 KEEP_MUTATIONS stage 需要复验后返回对性能有影响(很小)
SORT 情况
sort 情况顺序不可变,不可出现 AC 此时无法使用索引排序进入 SORT stage 是、排序效率降低。
即A存在在查询方面可无视对象排序,只排序时关注索引定义。prefix原则是只要存在index首字段即可并不需要考虑位置。
正文:
以公司的测试服务器的数据做实验 ,执行db.collection.getIndexes() 方法,此文档有大约27万数据。我们使用第8个复合索引作为测试索引
{ "v" : 1, "key" : { "state" : 1, "user" : 1, "stock" : 1 }, "name" : "state_1_user_1_stock_1", "ns" : "mango_order.od_reservations", "background" : true }
我们来看最佳 explain 即ABC db.od_reservations.find({state:1,user:32432424,stock:3243242}).explain(“executionStats”)
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "state" : { "$eq" : 1 } }, { "stock" : { "$eq" : 3243242 } } ] }, "winningPlan" : { "stage" : "FETCH", "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[3243242.0, 3243242.0]" ] } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 157, "totalKeysExamined" : 29806, "totalDocsExamined" : 0, "executionStages" : { "stage" : "FETCH", "nReturned" : 0, "executionTimeMillisEstimate" : 160, "works" : 29806, "advanced" : 0, "needTime" : 29805, "needYield" : 0, "saveState" : 232, "restoreState" : 232, "isEOF" : 1, "invalidates" : 0, "docsExamined" : 0, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 0, "executionTimeMillisEstimate" : 160, "works" : 29806, "advanced" : 0, "needTime" : 29805, "needYield" : 0, "saveState" : 232, "restoreState" : 232, "isEOF" : 1, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[3243242.0, 3243242.0]" ] }, "keysExamined" : 29806, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
接下来尝试 A_C 检索 db.od_reservations.find({state:1,stock:3243242}).explain(“executionStats”) 证明 A_C 使用该复合索引
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "state" : { "$eq" : 1 } }, { "stock" : { "$eq" : 3243242 } } ] }, "winningPlan" : { "stage" : "FETCH", "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[3243242.0, 3243242.0]" ] } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 157, "totalKeysExamined" : 29806, "totalDocsExamined" : 0, "executionStages" : { "stage" : "FETCH", "nReturned" : 0, "executionTimeMillisEstimate" : 160, "works" : 29806, "advanced" : 0, "needTime" : 29805, "needYield" : 0, "saveState" : 232, "restoreState" : 232, "isEOF" : 1, "invalidates" : 0, "docsExamined" : 0, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 0, "executionTimeMillisEstimate" : 160, "works" : 29806, "advanced" : 0, "needTime" : 29805, "needYield" : 0, "saveState" : 232, "restoreState" : 232, "isEOF" : 1, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[3243242.0, 3243242.0]" ] }, "keysExamined" : 29806, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
接下来验证 _BC db.od_reservations.find({user:32432424,stock:3243242}).explain(“executionStats”) 证明 _BC 不使用索引
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "stock" : { "$eq" : 3243242 } }, { "user" : { "$eq" : 32432424 } } ] }, "winningPlan" : { "stage" : "COLLSCAN", "filter" : { "$and" : [ { "stock" : { "$eq" : 3243242 } }, { "user" : { "$eq" : 32432424 } } ] }, "direction" : "forward" }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 92, "totalKeysExamined" : 0, "totalDocsExamined" : 277389, "executionStages" : { "stage" : "COLLSCAN", "filter" : { "$and" : [ { "stock" : { "$eq" : 3243242 } }, { "user" : { "$eq" : 32432424 } } ] }, "nReturned" : 0, "executionTimeMillisEstimate" : 80, "works" : 277401, "advanced" : 0, "needTime" : 277390, "needYield" : 10, "saveState" : 2171, "restoreState" : 2171, "isEOF" : 1, "invalidates" : 0, "direction" : "forward", "docsExamined" : 277389 } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
接下来验证 BA 是否使用索引 db.od_reservations.find({user:32432424,state:1}).explain(“executionStats”) 证明DBA正确,此时使用了复合索引 我们接下来看看和AB是否有性能差距
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "state" : { "$eq" : 1 } }, { "user" : { "$eq" : 32432424 } } ] }, "winningPlan" : { "stage" : "FETCH", "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 0, "totalKeysExamined" : 0, "totalDocsExamined" : 0, "executionStages" : { "stage" : "FETCH", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 1, "advanced" : 0, "needTime" : 0, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "docsExamined" : 0, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 1, "advanced" : 0, "needTime" : 0, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[1.0, 1.0]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] }, "keysExamined" : 0, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
验证 AB db.od_reservations.find({state:1,user:32432424}).explain(“executionStats”) 证明 BA与AB索引效率相同,AB可高效使用索引而AC则不行。
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "state" : { "$eq" : "1703282329098" } }, { "user" : { "$eq" : 32432424 } } ] }, "winningPlan" : { "stage" : "FETCH", "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[\"1703282329098\", \"1703282329098\"]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 0, "totalKeysExamined" : 0, "totalDocsExamined" : 0, "executionStages" : { "stage" : "FETCH", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 1, "advanced" : 0, "needTime" : 0, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "docsExamined" : 0, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 1, "advanced" : 0, "needTime" : 0, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[\"1703282329098\", \"1703282329098\"]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] }, "keysExamined" : 0, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
验证 $exists A:$exists—B db.od_reservations.find({state:{$exists: true},user:32432424}).explain(“executionStats”) 结论 $exites 返回多结果对性能有影响(
KEEP_UTATIONS stage 表示检索到结果后,在检索过程中的删除或者修改,会触发一个复验的过程,当复验成功文档将被返回
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "user" : { "$eq" : 32432424 } }, { "state" : { "$exists" : true } } ] }, "winningPlan" : { "stage" : "KEEP_MUTATIONS", "inputStage" : { "stage" : "FETCH", "filter" : { "state" : { "$exists" : true } }, "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[MinKey, MaxKey]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] } } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : true, "nReturned" : 0, "executionTimeMillis" : 0, "totalKeysExamined" : 3, "totalDocsExamined" : 0, "executionStages" : { "stage" : "KEEP_MUTATIONS", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 4, "advanced" : 0, "needTime" : 3, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "inputStage" : { "stage" : "FETCH", "filter" : { "state" : { "$exists" : true } }, "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 4, "advanced" : 0, "needTime" : 3, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "docsExamined" : 0, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 0, "executionTimeMillisEstimate" : 0, "works" : 4, "advanced" : 0, "needTime" : 3, "needYield" : 0, "saveState" : 0, "restoreState" : 0, "isEOF" : 1, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[MinKey, MaxKey]" ], "user" : [ "[32432424.0, 32432424.0]" ], "stock" : [ "[MinKey, MaxKey]" ] }, "keysExamined" : 3, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }
sort情况 db.od_reservations.find({state:{$exists:true},user:{$exists:true},stock:{$exists:true}}).sort({ state:1, stock:1}).explain(“executionStats”)
进行多种测试证明 sort排序时不能使用 AC BC 等索引 属于 SORT stage
{ "queryPlanner" : { "plannerVersion" : 1, "namespace" : "mango_order.od_reservations", "indexFilterSet" : false, "parsedQuery" : { "$and" : [ { "state" : { "$exists" : true } }, { "stock" : { "$exists" : true } }, { "user" : { "$exists" : true } } ] }, "winningPlan" : { "stage" : "SORT", "sortPattern" : { "state" : 1, "stock" : 1 }, "inputStage" : { "stage" : "KEEP_MUTATIONS", "inputStage" : { "stage" : "SORT_KEY_GENERATOR", "inputStage" : { "stage" : "FETCH", "filter" : { "$and" : [ { "state" : { "$exists" : true } }, { "user" : { "$exists" : true } }, { "stock" : { "$exists" : true } } ] }, "inputStage" : { "stage" : "IXSCAN", "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[MinKey, MaxKey]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[MinKey, MaxKey]" ] } } } } } }, "rejectedPlans" : [ ] }, "executionStats" : { "executionSuccess" : false, "errorMessage" : "Exec error: OperationFailed: Sort operation used more than the maximum 33554432 bytes of RAM. Add an index, or specify a smaller limit., state: FAILURE", "errorCode" : 96, "nReturned" : 0, "executionTimeMillis" : 313, "totalKeysExamined" : 101140, "totalDocsExamined" : 101140, "executionStages" : { "stage" : "SORT", "nReturned" : 0, "executionTimeMillisEstimate" : 310, "works" : 101142, "advanced" : 0, "needTime" : 101141, "needYield" : 0, "saveState" : 790, "restoreState" : 790, "isEOF" : 0, "invalidates" : 0, "sortPattern" : { "state" : 1, "stock" : 1 }, "memUsage" : 33554524, "memLimit" : 33554432, "inputStage" : { "stage" : "KEEP_MUTATIONS", "nReturned" : 101140, "executionTimeMillisEstimate" : 260, "works" : 101141, "advanced" : 101140, "needTime" : 1, "needYield" : 0, "saveState" : 790, "restoreState" : 790, "isEOF" : 0, "invalidates" : 0, "inputStage" : { "stage" : "SORT_KEY_GENERATOR", "nReturned" : 0, "executionTimeMillisEstimate" : 260, "works" : 101141, "advanced" : 0, "needTime" : 1, "needYield" : 0, "saveState" : 790, "restoreState" : 790, "isEOF" : 0, "invalidates" : 0, "inputStage" : { "stage" : "FETCH", "filter" : { "$and" : [ { "state" : { "$exists" : true } }, { "user" : { "$exists" : true } }, { "stock" : { "$exists" : true } } ] }, "nReturned" : 101140, "executionTimeMillisEstimate" : 200, "works" : 101140, "advanced" : 101140, "needTime" : 0, "needYield" : 0, "saveState" : 790, "restoreState" : 790, "isEOF" : 0, "invalidates" : 0, "docsExamined" : 101140, "alreadyHasObj" : 0, "inputStage" : { "stage" : "IXSCAN", "nReturned" : 101140, "executionTimeMillisEstimate" : 40, "works" : 101140, "advanced" : 101140, "needTime" : 0, "needYield" : 0, "saveState" : 790, "restoreState" : 790, "isEOF" : 0, "invalidates" : 0, "keyPattern" : { "state" : 1, "user" : 1, "stock" : 1 }, "indexName" : "state_1_user_1_stock_1", "isMultiKey" : false, "isUnique" : false, "isSparse" : false, "isPartial" : false, "indexVersion" : 1, "direction" : "forward", "indexBounds" : { "state" : [ "[MinKey, MaxKey]" ], "user" : [ "[MinKey, MaxKey]" ], "stock" : [ "[MinKey, MaxKey]" ] }, "keysExamined" : 101140, "dupsTested" : 0, "dupsDropped" : 0, "seenInvalidated" : 0 } } } } } }, "serverInfo" : { "host" : "mango", "port" : 27017, "version" : "3.2.8", "gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0" }, "ok" : 1 }