索引相交查询与COLLSCAN执行相同
我正在浏览一个相当古老的MongoDB数据库的现有索引,该数据库最近已从2.something升级到3.4。
访问它的代码是复杂的,自动的和面条意义重大的,再加上它允许面向Web的前端代码执行任意查询,所以很难概括地了解查询的具体内容。 做了一些分析后,我很快发现很多查询缺少索引,或者是因为逻辑索引丢失或拼写错误(是的!),或者因为有人假定复合索引会支持查询它们的非前缀部分。
这样的一个例子是users
集合,其中复合索引{ boxes._id: 1, _deleted: 1 }
被大量由{ _deleted: { $exists: false } }
过滤的查询漏掉了。
我对它进行了擦除,并创建了单字段索引,并阅读了索引交集功能,该功能现在是支持此应用程序的MongoDB服务器版本的一部分:
db.users.dropIndex({ 'boxes._id': 1, _deleted: 1 });
db.users.ensureIndex({ 'boxes._id': 1 });
db.users.ensureIndex({ '_deleted': 1 });
到现在为止还挺好。 这里的数据集不是很大,所以我没有看到直接的问题,在性能方面,通过删除复合索引和支持索引交集来代替。 AFAIU从文档和interwebs,使用复合索引性能收益大部分感觉与真正的大集合,对于这一个,我们正在看大约11K文件。
对两个索引字段中的任何一个进行简单的计数查询,在产生密钥时产生令人满意的响应:
db.runCommand({ explain: { count: 'users', query: { 'boxes._id': ObjectId('597745846ca2582d8b364c38') }, verbosity: 'executionStats' }})
// "executionTimeMillis" : 8
db.runCommand({ explain: { count: 'users', query: { _deleted: { $exists: false } }, verbosity: 'executionStats' }})
// "executionTimeMillis" : 35
(请注意,这是在我的旧MacBook Air上,并且与不使用explain
的性能日志中查看的查询时间相比,那么“正常”响应时间似乎低于在mongo中explain
报告的响应时间的25%贝壳)
但是,如果我对这两个字段进行过滤,从而触发交叉索引的使用,我会看到一个巨大的性能损失:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : { "_deleted" : [ "[null, null]" ] }
}
},
"rejectedPlans" : [
{
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
}
},
{
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"inputStage" : {
"stage" : "AND_SORTED",
"inputStages" : [
{
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
},
{
"stage" : "IXSCAN",
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
}
}
]
}
}
]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 11098,
"executionTimeMillis" : 731,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 702,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 127,
"restoreState" : 127,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 49,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 127,
"restoreState" : 127,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
},
"allPlansExecution" : [
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 101,
"seeks" : 1,
"dupsTested" : 101,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
{
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"totalKeysExamined" : 101,
"totalDocsExamined" : 50,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 101,
"advanced" : 50,
"needTime" : 51,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 50,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "AND_SORTED",
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 101,
"advanced" : 50,
"needTime" : 51,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"flagged" : 0,
"failedAnd_0" : 0,
"failedAnd_1" : 0,
"inputStages" : [
{
"stage" : "IXSCAN",
"nReturned" : 51,
"executionTimeMillisEstimate" : 0,
"works" : 51,
"advanced" : 51,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 51,
"seeks" : 1,
"dupsTested" : 51,
"dupsDropped" : 0,
"seenInvalidated" : 0
},
{
"stage" : "IXSCAN",
"nReturned" : 50,
"executionTimeMillisEstimate" : 12,
"works" : 50,
"advanced" : 50,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 50,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
]
}
}
},
{
"nReturned" : 101,
"executionTimeMillisEstimate" : 11,
"totalKeysExamined" : 101,
"totalDocsExamined" : 101,
"executionStages" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
}
]
},
"nReturned" : 101,
"executionTimeMillisEstimate" : 11,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101,
"executionTimeMillisEstimate" : 0,
"works" : 101,
"advanced" : 101,
"needTime" : 0,
"needYield" : 0,
"saveState" : 3,
"restoreState" : 3,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : { "_deleted" : 1 },
"indexName" : "_deleted_1",
"isMultiKey" : false,
"multiKeyPaths" : { "_deleted" : [ ] },
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 101,
"seeks" : 1,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
]
}
}
相同的查询命中复合索引:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1,
"_deleted" : 1
},
"indexName" : "boxes._id_1__deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
],
"_deleted" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
],
"_deleted" : [ "[null, null]" ]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 37,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 35,
"works" : 11099,
"advanced" : 0,
"needTime" : 11098,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 35,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 23,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 87,
"restoreState" : 87,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1,
"_deleted" : 1
},
"indexName" : "boxes._id_1__deleted_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
],
"_deleted" : [ ]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
],
"_deleted" : [ "[null, null]" ]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 11098,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"allPlansExecution" : [ ]
}
}
用boxes._id
上的单个索引替换复合索引的性能与在两个已过滤字段上使用复合键的性能几乎相同:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"boxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 31,
"totalKeysExamined" : 11098,
"totalDocsExamined" : 11098,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 34,
"works" : 11099,
"advanced" : 0,
"needTime" : 11098,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 34,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 11098,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 11098,
"executionTimeMillisEstimate" : 11,
"works" : 11099,
"advanced" : 11098,
"needTime" : 0,
"needYield" : 0,
"saveState" : 88,
"restoreState" : 88,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"boxes._id" : 1
},
"indexName" : "boxes._id_1",
"isMultiKey" : true,
"multiKeyPaths" : {
"boxes._id" : [
"boxes"
]
},
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 2,
"direction" : "forward",
"indexBounds" : {
"boxes._id" : [
"[ObjectId('597745846ca2582d8b364c38'), ObjectId('597745846ca2582d8b364c38')]"
]
},
"keysExamined" : 11098,
"seeks" : 1,
"dupsTested" : 11098,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"allPlansExecution" : [ ]
}
}
最后,删除这些字段上的所有索引,触发COLLSCAN
性能与索引相交性能几乎相同:
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "bl.users",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"winningPlan" : {
"stage" : "COUNT",
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"direction" : "forward"
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 848,
"totalKeysExamined" : 0,
"totalDocsExamined" : 11109,
"executionStages" : {
"stage" : "COUNT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 847,
"works" : 11111,
"advanced" : 0,
"needTime" : 11110,
"needYield" : 0,
"saveState" : 123,
"restoreState" : 123,
"isEOF" : 1,
"invalidates" : 0,
"nCounted" : 11098,
"nSkipped" : 0,
"inputStage" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"ideaboxes._id" : {
"$eq" : ObjectId("597745846ca2582d8b364c38")
}
},
{
"$nor" : [ { "_deleted" : { "$exists" : true } } ]
}
]
},
"nReturned" : 11098,
"executionTimeMillisEstimate" : 847,
"works" : 11111,
"advanced" : 11098,
"needTime" : 12,
"needYield" : 0,
"saveState" : 123,
"restoreState" : 123,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 11109
}
},
"allPlansExecution" : [ ]
}
}
当然,通过索引交叉查询获得COLLSCAN性能是不对的? 我承认我对Mongo非常陌生,所以当我试图破译这份报告时,我也很无知 - 答案可能正在盯着我。
我已经在一个ObjectRocket实例上对Mongo 3.2上的相同查询进行了测试,并且他们的服务器比我老旧的笔记本电脑快得多,但他们相对来说显示了相同的数字。
任何想法为什么发生这种情况?
谢谢 :)
链接地址: http://www.djcxy.com/p/57919.html上一篇: Index intersection queries performing the same as COLLSCAN
下一篇: How to speed up a query which use several string filters?