sequelize: Limit causing "missing FROM-clause entry for table"

Hi,

Got an issue with sequelize, tried 3.5 and 3.7.1, using postgres. I added as much detail as I could below, but the short version is that adding limit to a query with an include results in a different query than expected.

I simplified the models for readability, but its just a simple job system. Lets say the job is to run echo 'Hello World' than a Run would be status/result of the execution. You can run a job multiple times resulting in a new Run model. During execution the system may add RunMessages to the Run, which can be the output of the command.

What I’m trying to query is the answer to the following question: For a given job what are the last 10 runs and the number of messages for each run

The result should be something like

[
 {"id":"...","status": "done","jobid":"...", ... , "msg_count":3},
 {"id":"...","status": "done","jobid":"...", ... , "msg_count":9},
 {"id":"...","status": "done","jobid":"...", ... , "msg_count":0},
 ...
]

Working query without limit

Query options:

{
  "where" : {
    "jobid":req.params.jobid
  },
  "attributes":
    Object.keys(this.db.model("Run").attributes).concat([
      [sequelize.fn('COUNT',sequelize.col('messages.id')),"msg_count"]
    ]),
  "include":[
      {
        "model": this.db.model("RunMessage"),
        "as" : "messages",
        "attributes":[]
      }
  ],
  "group":['"Run.id"'],
  "order":[["createdAt","DESC"]],
  "logging": console.log,
}

Query

SELECT 
  "Run"."id", 
  "Run"."status", 
  "Run"."jobid",
  "Run"."createdAt", 
  "Run"."updatedAt", 
  COUNT("messages"."id") AS "msg_count"
FROM "Runs" AS "Run" 
LEFT OUTER JOIN "RunMessages" AS "messages" ON "Run"."id" = "messages"."runid" 
WHERE "Run"."jobid" = '607169ba-e6ec-4efa-8aca-b70ffc1d2ffb' 
GROUP BY "Run"."id" 
ORDER BY "Run"."createdAt" DESC;

Adding Limit

{
  "where" : {
    "jobid":req.params.jobid
  },
  "attributes":
    Object.keys(this.db.model("Run").attributes).concat([
      [sequelize.fn('COUNT',sequelize.col('messages.id')),"msg_count"]
    ]),
  "include":[
      {
        "model": this.db.model("RunMessage"),
        "as" : "messages",
        "attributes":[]
      }
  ],
  "group":['"Run.id"'],
  "order":[["createdAt","DESC"]],
  "logging": console.log,
  "limit": 10
}

Query

SELECT 
  "Run".* 
FROM (
  SELECT 
    "Run"."id", 
    "Run"."status", 
    "Run"."jobid", 
    "Run"."createdAt", 
    "Run"."updatedAt", 
    COUNT("messages"."id") AS "msg_count" 
  FROM "Runs" AS "Run" 
  WHERE "Run"."jobid" = '607169ba-e6ec-4efa-8aca-b70ffc1d2ffb' 
  GROUP BY "Run"."id"
  ORDER BY "Run"."createdAt" DESC
  LIMIT 10
) AS "Run" 
LEFT OUTER JOIN "RunMessages" AS "messages" ON "Run"."id" = "messages"."runid"
ORDER BY "Run"."createdAt" DESC;

Error

SequelizeDatabaseError: missing FROM-clause entry for table "messages"
job_1 |     at Query.formatError (/api-jobs/node_modules/sequelize/lib/dialects/postgres/query.js:433:14)
job_1 |     at null.<anonymous> (/api-jobs/node_modules/sequelize/lib/dialects/postgres/query.js:108:19)
job_1 |     at emit (events.js:107:17)
job_1 |     at Query.handleError (/regressinator-api-jobs/node_modules/pg/lib/query.js:99:8)
job_1 |     at null.<anonymous> (/regressinator-api-jobs/node_modules/pg/lib/client.js:166:26)
job_1 |     at emit (events.js:107:17)
job_1 |     at Socket.<anonymous> (/regressinator-api-jobs/node_modules/pg/lib/connection.js:109:12)
job_1 |     at Socket.emit (events.js:107:17)
job_1 |     at readableAddChunk (_stream_readable.js:163:16)
job_1 |     at Socket.Readable.push (_stream_readable.js:126:10)
job_1 |     at TCP.onread (net.js:538:20)

Expected Query

SELECT 
  "Run"."id", 
  "Run"."status", 
  "Run"."jobid",
  "Run"."createdAt", 
  "Run"."updatedAt", 
  COUNT("messages"."id") AS "msg_count"
FROM "Runs" AS "Run" 
LEFT OUTER JOIN "RunMessages" AS "messages" ON "Run"."id" = "messages"."runid" 
WHERE "Run"."jobid" = '607169ba-e6ec-4efa-8aca-b70ffc1d2ffb' 
GROUP BY "Run"."id" 
ORDER BY "Run"."createdAt" DESC
LIMIT 10;

Models

Run

db.define(
    // Model name
    "Run",
    // Properties
    /** @lends Run.prototype */
    {
      /**
      * Identifier of the Job/Run
      * @readonly
      * @type {UUID}
      */
      id : {
        type: DataTypes.UUID,
        //Use V4 UUIDs
        defaultValue: DataTypes.UUIDV4,
        allowNull: false,
        primaryKey: true
      },
      /**
      * Actual status of the job
      * @type {"created"|"running"|"done"|"failed"}
      */
      status : {
        type: DataTypes.ENUM("created","running","done","failed"),
        defaultValue: "created",
        allowNull: false
      },
      /**
      * Job that we should run
      * @type {UUID}
      */
      jobid : {
        type: DataTypes.UUID,
        allowNull:false
      },
    }
  );

RunMessage

db.define(
    // Model name
    "RunMessage",
    // Properties
    /** @lends RunMessage.prototype */
    {
      /**
      * Identifier of the Job/RunMessage
      * @readonly
      * @type {UUID}
      */
      id : {
        type: DataTypes.UUID,
        //Use V4 UUIDs
        defaultValue: DataTypes.UUIDV4,
        allowNull: false,
        primaryKey: true
      },
      /**
      * Run associated with this RunMessage
      * @type {UUID}
      */
      runid : {
        type: DataTypes.UUID,
        allowNull:false
      },
      /**
      * A textual description of the notification
      *
      * @type {Text}
      */
      message : {
        type: DataTypes.TEXT,
        allowNull:false
      },
    },
  );

Association

db.model("Run").hasMany(
  db.model("RunMessage"),
  {
    "as" : "messages",
    "foreignKey" : "runid",
    "onDelete" : "cascade",
    "hooks" : true
  }
);

About this issue

  • Original URL
  • State: closed
  • Created 9 years ago
  • Reactions: 15
  • Comments: 20 (6 by maintainers)

Most upvoted comments

You can try using include.duplicating = false in that case.

duplicating: false fixed my query but I’m still left very confused. What does this option do? I can’t seem to find it in the docs anywhere… hmm

@mickhansen duplicating: false breaks limit and offset

That did the trick!

Is there any place where these hidden ‘features’ are documented? Readthedocs and even the source code don’t specify the behaviour of duplicating.

Closing ticket as its resolved.

Final Query options

{
  "where" : {
    "jobid":req.params.jobid
  },
  "attributes":
    Object.keys(this.db.model("Run").attributes).concat([
      [sequelize.fn('COUNT',sequelize.col('messages.id')),"msg_count"]
    ]),
  "include":[
      {
        "model": this.db.model("RunMessage"),
        "as" : "messages",
        "attributes":[],
        "duplicating":false
      }
  ],
  "group":['"Run.id"'],
  "order":[["createdAt","DESC"]],
  "logging": console.log,
}

Final Query

SELECT 
  "Run"."id", 
  "Run"."status", 
  "Run"."jobid",
  "Run"."createdAt", 
  "Run"."updatedAt", 
  COUNT("messages"."id") AS "msg_count" 
FROM "Runs" AS "Run" 
LEFT OUTER JOIN "RunMessages" AS "messages" ON "Run"."id" = "messages"."runid" 
WHERE "Run"."jobid" = '607169ba-e6ec-4efa-8aca-b70ffc1d2ffb' 
GROUP BY "Run"."id" 
ORDER BY "Run"."createdAt" DESC 
LIMIT 10;

If above didn’t work try this! The only way I could get this to work was to do subQuery: false when I had 2 nested includes (joins) with server side paging/sorting/filtering and reporting row counts. It still reports the row count wrong when I do findAndCountAll if the nested joins dupe up across a 1-M but that might be bareable (not really easy to fix that I don’t think w/o a more expensive query!)

subQuery: false gives incorrect data when limit or offset are used on M:N relations

Interesting, adding duplicating=false to my include options make sequelize generate the right query. Anyone have any idea why this is the case?

I tried the required but it returns the same issue via a different SQL query. If the new query did work it also would return an incorrect result as all runs should be visible even without messages.

Don’t fully understand why a subquery is needed for limit as its a left outer join. The expected query works and uses limit correctly. It might be required to a use-case I’m not aware of.

Will execute the expected query natively to solve it for now.

Code changed

"include":[
  {
    "model": this.db.model("RunMessage"),
    "as" : "messages",
    "attributes":[],
    "required": true
  }
]

Query

SELECT 
  "Run".* 
FROM (
  SELECT
    "Run"."id",
    "Run"."status", 
    "Run"."jobid", 
    "Run"."createdAt", 
    "Run"."updatedAt", COUNT("messages"."id") AS "msg_count" 
  FROM "Runs" AS "Run" 
  WHERE 
    "Run"."jobid" = '607169ba-e6ec-4efa-8aca-b70ffc1d2ffb' 
    AND (
      SELECT 
        "runid" 
      FROM "RunMessages" AS "RunMessage" 
      WHERE ("RunMessage"."runid" = "Run"."id") 
      LIMIT 1
    ) IS NOT NULL
  GROUP BY "Run"."id" 
  ORDER BY "Run"."createdAt" DESC 
  LIMIT 10) AS "Run" 
INNER JOIN "RunMessages" AS "messages" ON "Run"."id" = "messages"."runid" 
ORDER BY "Run"."createdAt" DESC;

For limit we need a subquery, and in that case [sequelize.fn('COUNT',sequelize.col('messages.id')),"msg_count"] won’t be in the right scope.

No easy way to fix this other than set include.required = true on the messages include.