sched.h 52.1 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

5
6
7
8
/*
 * Define 'struct task_struct' and provide the main scheduler
 * APIs (schedule(), wakeup variants, etc.)
 */
9

10
#include <uapi/linux/sched.h>
11

12
#include <asm/current.h>
Linus Torvalds's avatar
Linus Torvalds committed
13

14
#include <linux/pid.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/sem.h>
16
#include <linux/shm.h>
17
18
19
20
#include <linux/kcov.h>
#include <linux/mutex.h>
#include <linux/plist.h>
#include <linux/hrtimer.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
#include <linux/seccomp.h>
22
#include <linux/nodemask.h>
23
#include <linux/rcupdate.h>
24
#include <linux/resource.h>
Arjan van de Ven's avatar
Arjan van de Ven committed
25
#include <linux/latencytop.h>
26
27
28
29
#include <linux/sched/prio.h>
#include <linux/signal_types.h>
#include <linux/mm_types_task.h>
#include <linux/task_io_accounting.h>
30
#include <linux/rseq.h>
31

32
/* task_struct member predeclarations (sorted alphabetically): */
33
34
struct audit_context;
struct backing_dev_info;
35
struct bio_list;
36
struct blk_plug;
37
38
39
40
41
struct cfs_rq;
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct mempolicy;
42
struct nameidata;
43
44
45
46
47
48
49
50
51
struct nsproxy;
struct perf_event_context;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
struct robust_list_head;
struct sched_attr;
struct sched_param;
Ingo Molnar's avatar
Ingo Molnar committed
52
struct seq_file;
53
54
55
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
56
struct task_group;
Linus Torvalds's avatar
Linus Torvalds committed
57

58
59
60
61
62
63
64
65
66
67
/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */
68
69

/* Used in tsk->state: */
70
71
72
73
74
#define TASK_RUNNING			0x0000
#define TASK_INTERRUPTIBLE		0x0001
#define TASK_UNINTERRUPTIBLE		0x0002
#define __TASK_STOPPED			0x0004
#define __TASK_TRACED			0x0008
75
/* Used in tsk->exit_state: */
76
77
#define EXIT_DEAD			0x0010
#define EXIT_ZOMBIE			0x0020
78
79
#define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
/* Used in tsk->state again: */
80
81
82
83
#define TASK_PARKED			0x0040
#define TASK_DEAD			0x0080
#define TASK_WAKEKILL			0x0100
#define TASK_WAKING			0x0200
84
85
86
#define TASK_NOLOAD			0x0400
#define TASK_NEW			0x0800
#define TASK_STATE_MAX			0x1000
87
88
89
90
91
92
93
94
95
96
97
98
99
100

/* Convenience macros for the sake of set_current_state: */
#define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)

#define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

/* Convenience macros for the sake of wake_up(): */
#define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)

/* get_task_state(): */
#define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
101
102
					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
					 TASK_PARKED)
103
104
105
106
107
108
109
110
111
112

#define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)

#define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)

#define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)

#define task_contributes_to_load(task)	((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
					 (task->flags & PF_FROZEN) == 0 && \
					 (task->state & TASK_NOLOAD) == 0)
Linus Torvalds's avatar
Linus Torvalds committed
113

Peter Zijlstra's avatar
Peter Zijlstra committed
114
115
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

116
117
118
119
120
121
122
/*
 * Special states are those that do not use the normal wait-loop pattern. See
 * the comment with set_special_state().
 */
#define is_special_task_state(state)				\
	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_DEAD))

Peter Zijlstra's avatar
Peter Zijlstra committed
123
124
#define __set_current_state(state_value)			\
	do {							\
125
		WARN_ON_ONCE(is_special_task_state(state_value));\
Peter Zijlstra's avatar
Peter Zijlstra committed
126
127
128
		current->task_state_change = _THIS_IP_;		\
		current->state = (state_value);			\
	} while (0)
129

Peter Zijlstra's avatar
Peter Zijlstra committed
130
131
#define set_current_state(state_value)				\
	do {							\
132
		WARN_ON_ONCE(is_special_task_state(state_value));\
Peter Zijlstra's avatar
Peter Zijlstra committed
133
		current->task_state_change = _THIS_IP_;		\
134
		smp_store_mb(current->state, (state_value));	\
Peter Zijlstra's avatar
Peter Zijlstra committed
135
136
	} while (0)

137
138
139
140
141
142
143
144
145
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		WARN_ON_ONCE(!is_special_task_state(state_value));	\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->task_state_change = _THIS_IP_;			\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)
Peter Zijlstra's avatar
Peter Zijlstra committed
146
#else
147
148
149
150
151
/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
152
 *   for (;;) {
153
 *	set_current_state(TASK_UNINTERRUPTIBLE);
154
155
156
157
158
159
160
161
162
163
164
165
166
 *	if (!need_sleep)
 *		break;
 *
 *	schedule();
 *   }
 *   __set_current_state(TASK_RUNNING);
 *
 * If the caller does not need such serialisation (because, for instance, the
 * condition test and condition change and wakeup are under the same lock) then
 * use __set_current_state().
 *
 * The above is typically ordered against the wakeup, which does:
 *
167
168
 *   need_sleep = false;
 *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
169
170
171
172
173
174
175
 *
 * Where wake_up_state() (and all other wakeup primitives) imply enough
 * barriers to order the store of the variable against wakeup.
 *
 * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
176
 *
177
178
179
180
 * However, with slightly different timing the wakeup TASK_RUNNING store can
 * also collide with the TASK_UNINTERRUPTIBLE store. Loosing that store is not
 * a problem either because that will result in one extra go around the loop
 * and our @cond test will save the day.
181
 *
182
 * Also see the comments of try_to_wake_up().
183
 */
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
#define __set_current_state(state_value)				\
	current->state = (state_value)

#define set_current_state(state_value)					\
	smp_store_mb(current->state, (state_value))

/*
 * set_special_state() should be used for those states when the blocking task
 * can not use the regular condition based wait-loop. In that case we must
 * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
 * will not collide with our state change.
 */
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)

Peter Zijlstra's avatar
Peter Zijlstra committed
204
205
#endif

206
207
/* Task command name length: */
#define TASK_COMM_LEN			16
Linus Torvalds's avatar
Linus Torvalds committed
208
209
210

extern void scheduler_tick(void);

211
212
213
214
215
216
217
#define	MAX_SCHEDULE_TIMEOUT		LONG_MAX

extern long schedule_timeout(long timeout);
extern long schedule_timeout_interruptible(long timeout);
extern long schedule_timeout_killable(long timeout);
extern long schedule_timeout_uninterruptible(long timeout);
extern long schedule_timeout_idle(long timeout);
Linus Torvalds's avatar
Linus Torvalds committed
218
asmlinkage void schedule(void);
219
extern void schedule_preempt_disabled(void);
Linus Torvalds's avatar
Linus Torvalds committed
220

221
222
extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
223
extern long io_schedule_timeout(long timeout);
224
extern void io_schedule(void);
225

226
/**
227
 * struct prev_cputime - snapshot of system and user cputime
228
229
 * @utime: time spent in user mode
 * @stime: time spent in system mode
230
 * @lock: protects the above two fields
231
 *
232
233
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
234
 */
235
236
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
237
238
239
	u64				utime;
	u64				stime;
	raw_spinlock_t			lock;
240
#endif
241
242
};

243
244
/**
 * struct task_cputime - collected CPU time counts
245
246
 * @utime:		time spent in user mode, in nanoseconds
 * @stime:		time spent in kernel mode, in nanoseconds
247
 * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
248
 *
249
250
251
 * This structure groups together three kinds of CPU time that are tracked for
 * threads and thread groups.  Most things considering CPU time want to group
 * these counts together and treat all three of them in parallel.
252
253
 */
struct task_cputime {
254
255
256
	u64				utime;
	u64				stime;
	unsigned long long		sum_exec_runtime;
257
};
258

259
260
261
262
/* Alternate field names when used on cache expirations: */
#define virt_exp			utime
#define prof_exp			stime
#define sched_exp			sum_exec_runtime
263

264
265
266
267
268
269
270
271
272
273
274
275
276
enum vtime_state {
	/* Task is sleeping or running in a CPU with VTIME inactive: */
	VTIME_INACTIVE = 0,
	/* Task runs in userspace in a CPU with VTIME active: */
	VTIME_USER,
	/* Task runs in kernelspace in a CPU with VTIME active: */
	VTIME_SYS,
};

struct vtime {
	seqcount_t		seqcount;
	unsigned long long	starttime;
	enum vtime_state	state;
277
278
279
	u64			utime;
	u64			stime;
	u64			gtime;
280
281
};

Linus Torvalds's avatar
Linus Torvalds committed
282
struct sched_info {
283
#ifdef CONFIG_SCHED_INFO
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
	/* Cumulative counters: */

	/* # of times we have run on this CPU: */
	unsigned long			pcount;

	/* Time spent waiting on a runqueue: */
	unsigned long long		run_delay;

	/* Timestamps: */

	/* When did we last run on a CPU? */
	unsigned long long		last_arrival;

	/* When were we last queued to run? */
	unsigned long long		last_queued;
Linus Torvalds's avatar
Linus Torvalds committed
299

300
#endif /* CONFIG_SCHED_INFO */
301
};
Linus Torvalds's avatar
Linus Torvalds committed
302

303
304
305
306
307
308
309
/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
310
311
# define SCHED_FIXEDPOINT_SHIFT		10
# define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
312

Ingo Molnar's avatar
Ingo Molnar committed
313
struct load_weight {
314
315
	unsigned long			weight;
	u32				inv_weight;
Ingo Molnar's avatar
Ingo Molnar committed
316
317
};

318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
/**
 * struct util_est - Estimation utilization of FAIR tasks
 * @enqueued: instantaneous estimated utilization of a task/cpu
 * @ewma:     the Exponential Weighted Moving Average (EWMA)
 *            utilization of a task
 *
 * Support data structure to track an Exponential Weighted Moving Average
 * (EWMA) of a FAIR task's utilization. New samples are added to the moving
 * average each time a task completes an activation. Sample's weight is chosen
 * so that the EWMA will be relatively insensitive to transient changes to the
 * task's workload.
 *
 * The enqueued attribute has a slightly different meaning for tasks and cpus:
 * - task:   the task's util_avg at last task dequeue time
 * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
 * Thus, the util_est.enqueued of a task represents the contribution on the
 * estimated utilization of the CPU where that task is currently enqueued.
 *
 * Only for tasks we track a moving average of the past instantaneous
 * estimated utilization. This allows to absorb sporadic drops in utilization
 * of an otherwise almost periodic task.
 */
struct util_est {
	unsigned int			enqueued;
	unsigned int			ewma;
#define UTIL_EST_WEIGHT_SHIFT		2
344
} __attribute__((__aligned__(sizeof(u64))));
345

346
/*
347
348
349
350
351
352
353
354
355
 * The load_avg/util_avg accumulates an infinite geometric series
 * (see __update_load_avg() in kernel/sched/fair.c).
 *
 * [load_avg definition]
 *
 *   load_avg = runnable% * scale_load_down(load)
 *
 * where runnable% is the time ratio that a sched_entity is runnable.
 * For cfs_rq, it is the aggregated load_avg of all runnable and
356
 * blocked sched_entities.
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
 *
 * load_avg may also take frequency scaling into account:
 *
 *   load_avg = runnable% * scale_load_down(load) * freq%
 *
 * where freq% is the CPU frequency normalized to the highest frequency.
 *
 * [util_avg definition]
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE
 *
 * where running% is the time ratio that a sched_entity is running on
 * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
 * and blocked sched_entities.
 *
 * util_avg may also factor frequency scaling and CPU capacity scaling:
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity%
 *
 * where freq% is the same as above, and capacity% is the CPU capacity
 * normalized to the greatest capacity (due to uarch differences, etc).
 *
 * N.B., the above ratios (runnable%, running%, freq%, and capacity%)
 * themselves are in the range of [0, 1]. To do fixed point arithmetics,
 * we therefore scale them to as large a range as necessary. This is for
 * example reflected by util_avg's SCHED_CAPACITY_SCALE.
 *
 * [Overflow issue]
 *
 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
 * with the highest load (=88761), always runnable on a single cfs_rq,
 * and should not overflow as the number already hits PID_MAX_LIMIT.
 *
 * For all other cases (including 32-bit kernels), struct load_weight's
 * weight will overflow first before we do, because:
 *
 *    Max(load_avg) <= Max(load.weight)
 *
 * Then it is the load_weight's responsibility to consider overflow
 * issues.
397
 */
398
struct sched_avg {
399
400
	u64				last_update_time;
	u64				load_sum;
401
	u64				runnable_load_sum;
402
403
404
	u32				util_sum;
	u32				period_contrib;
	unsigned long			load_avg;
405
	unsigned long			runnable_load_avg;
406
	unsigned long			util_avg;
407
	struct util_est			util_est;
408
} ____cacheline_aligned;
409

410
struct sched_statistics {
411
#ifdef CONFIG_SCHEDSTATS
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
	u64				wait_start;
	u64				wait_max;
	u64				wait_count;
	u64				wait_sum;
	u64				iowait_count;
	u64				iowait_sum;

	u64				sleep_start;
	u64				sleep_max;
	s64				sum_sleep_runtime;

	u64				block_start;
	u64				block_max;
	u64				exec_max;
	u64				slice_max;

	u64				nr_migrations_cold;
	u64				nr_failed_migrations_affine;
	u64				nr_failed_migrations_running;
	u64				nr_failed_migrations_hot;
	u64				nr_forced_migrations;

	u64				nr_wakeups;
	u64				nr_wakeups_sync;
	u64				nr_wakeups_migrate;
	u64				nr_wakeups_local;
	u64				nr_wakeups_remote;
	u64				nr_wakeups_affine;
	u64				nr_wakeups_affine_attempts;
	u64				nr_wakeups_passive;
	u64				nr_wakeups_idle;
443
#endif
444
};
445
446

struct sched_entity {
447
448
	/* For load-balancing: */
	struct load_weight		load;
449
	unsigned long			runnable_weight;
450
451
452
	struct rb_node			run_node;
	struct list_head		group_node;
	unsigned int			on_rq;
453

454
455
456
457
	u64				exec_start;
	u64				sum_exec_runtime;
	u64				vruntime;
	u64				prev_sum_exec_runtime;
458

459
	u64				nr_migrations;
460

461
	struct sched_statistics		statistics;
462

Ingo Molnar's avatar
Ingo Molnar committed
463
#ifdef CONFIG_FAIR_GROUP_SCHED
464
465
	int				depth;
	struct sched_entity		*parent;
Ingo Molnar's avatar
Ingo Molnar committed
466
	/* rq on which this entity is (to be) queued: */
467
	struct cfs_rq			*cfs_rq;
Ingo Molnar's avatar
Ingo Molnar committed
468
	/* rq "owned" by this entity/group: */
469
	struct cfs_rq			*my_q;
Ingo Molnar's avatar
Ingo Molnar committed
470
#endif
471

472
#ifdef CONFIG_SMP
473
474
475
476
477
478
	/*
	 * Per entity load average tracking.
	 *
	 * Put into separate cache line so it does not
	 * collide with read-mostly values above.
	 */
479
	struct sched_avg		avg;
480
#endif
Ingo Molnar's avatar
Ingo Molnar committed
481
};
482

Peter Zijlstra's avatar
Peter Zijlstra committed
483
struct sched_rt_entity {
484
485
486
487
488
489
490
491
	struct list_head		run_list;
	unsigned long			timeout;
	unsigned long			watchdog_stamp;
	unsigned int			time_slice;
	unsigned short			on_rq;
	unsigned short			on_list;

	struct sched_rt_entity		*back;
492
#ifdef CONFIG_RT_GROUP_SCHED
493
	struct sched_rt_entity		*parent;
Peter Zijlstra's avatar
Peter Zijlstra committed
494
	/* rq on which this entity is (to be) queued: */
495
	struct rt_rq			*rt_rq;
Peter Zijlstra's avatar
Peter Zijlstra committed
496
	/* rq "owned" by this entity/group: */
497
	struct rt_rq			*my_q;
Peter Zijlstra's avatar
Peter Zijlstra committed
498
#endif
499
} __randomize_layout;
Peter Zijlstra's avatar
Peter Zijlstra committed
500

501
struct sched_dl_entity {
502
	struct rb_node			rb_node;
503
504
505

	/*
	 * Original scheduling parameters. Copied here from sched_attr
506
507
	 * during sched_setattr(), they will remain the same until
	 * the next sched_setattr().
508
	 */
509
510
511
	u64				dl_runtime;	/* Maximum runtime for each instance	*/
	u64				dl_deadline;	/* Relative deadline of each instance	*/
	u64				dl_period;	/* Separation of two instances (period) */
512
	u64				dl_bw;		/* dl_runtime / dl_period		*/
513
	u64				dl_density;	/* dl_runtime / dl_deadline		*/
514
515
516
517
518
519

	/*
	 * Actual scheduling parameters. Initialized with the values above,
	 * they are continously updated during task execution. Note that
	 * the remaining runtime could be < 0 in case we are in overrun.
	 */
520
521
522
	s64				runtime;	/* Remaining runtime for this instance	*/
	u64				deadline;	/* Absolute deadline for this instance	*/
	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
523
524
525
526
527
528
529
530

	/*
	 * Some bool flags:
	 *
	 * @dl_throttled tells if we exhausted the runtime. If so, the
	 * task has to wait for a replenishment to be performed at the
	 * next firing of dl_timer.
	 *
531
532
	 * @dl_boosted tells if we are boosted due to DI. If so we are
	 * outside bandwidth enforcement mechanism (but only until we
533
534
	 * exit the critical section);
	 *
535
	 * @dl_yielded tells if task gave up the CPU before consuming
536
	 * all its available runtime during the last job.
537
538
539
540
541
542
543
	 *
	 * @dl_non_contending tells if the task is inactive while still
	 * contributing to the active utilization. In other words, it
	 * indicates if the inactive timer has been armed and its handler
	 * has not been executed yet. This flag is useful to avoid race
	 * conditions between the inactive timer handler and the wakeup
	 * code.
544
545
546
	 *
	 * @dl_overrun tells if the task asked to be informed about runtime
	 * overruns.
547
	 */
548
549
550
551
	unsigned int			dl_throttled      : 1;
	unsigned int			dl_boosted        : 1;
	unsigned int			dl_yielded        : 1;
	unsigned int			dl_non_contending : 1;
552
	unsigned int			dl_overrun	  : 1;
553
554
555
556
557

	/*
	 * Bandwidth enforcement timer. Each -deadline task has its
	 * own bandwidth to be enforced, thus we need one timer per task.
	 */
558
	struct hrtimer			dl_timer;
559
560
561
562
563
564
565
566
567

	/*
	 * Inactive timer, responsible for decreasing the active utilization
	 * at the "0-lag time". When a -deadline task blocks, it contributes
	 * to GRUB's active utilization until the "0-lag time", hence a
	 * timer is needed to decrease the active utilization at the correct
	 * time.
	 */
	struct hrtimer inactive_timer;
568
};
569

570
571
union rcu_special {
	struct {
572
573
574
575
576
577
		u8			blocked;
		u8			need_qs;
		u8			exp_need_qs;

		/* Otherwise the compiler can store garbage here: */
		u8			pad;
578
579
	} b; /* Bits. */
	u32 s; /* Set of bits. */
580
};
581

Peter Zijlstra's avatar
Peter Zijlstra committed
582
583
584
enum perf_event_task_context {
	perf_invalid_context = -1,
	perf_hw_context = 0,
585
	perf_sw_context,
Peter Zijlstra's avatar
Peter Zijlstra committed
586
587
588
	perf_nr_task_contexts,
};

589
590
591
592
struct wake_q_node {
	struct wake_q_node *next;
};

Linus Torvalds's avatar
Linus Torvalds committed
593
struct task_struct {
594
595
596
597
598
#ifdef CONFIG_THREAD_INFO_IN_TASK
	/*
	 * For reasons of header soup (see current_thread_info()), this
	 * must be the first element of task_struct.
	 */
599
	struct thread_info		thread_info;
600
#endif
601
602
	/* -1 unrunnable, 0 runnable, >0 stopped: */
	volatile long			state;
603
604
605
606
607
608
609

	/*
	 * This begins the randomizable portion of task_struct. Only
	 * scheduling-critical items should be added above here.
	 */
	randomized_struct_fields_start

610
611
612
613
614
	void				*stack;
	atomic_t			usage;
	/* Per task flags (PF_*), defined further below: */
	unsigned int			flags;
	unsigned int			ptrace;
Linus Torvalds's avatar
Linus Torvalds committed
615

616
#ifdef CONFIG_SMP
617
618
	struct llist_node		wake_entry;
	int				on_cpu;
619
#ifdef CONFIG_THREAD_INFO_IN_TASK
620
621
	/* Current CPU: */
	unsigned int			cpu;
622
#endif
623
624
625
	unsigned int			wakee_flips;
	unsigned long			wakee_flip_decay_ts;
	struct task_struct		*last_wakee;
626

627
628
629
630
631
632
633
634
	/*
	 * recent_used_cpu is initially set as the last CPU used by a task
	 * that wakes affine another task. Waker/wakee relationships can
	 * push tasks around a CPU where each wakeup moves to the next one.
	 * Tracking a recently used CPU allows a quick search for a recently
	 * used CPU that may be idle.
	 */
	int				recent_used_cpu;
635
	int				wake_cpu;
636
#endif
637
638
639
640
641
642
	int				on_rq;

	int				prio;
	int				static_prio;
	int				normal_prio;
	unsigned int			rt_priority;
643

644
645
646
	const struct sched_class	*sched_class;
	struct sched_entity		se;
	struct sched_rt_entity		rt;
647
#ifdef CONFIG_CGROUP_SCHED
648
	struct task_group		*sched_task_group;
649
#endif
650
	struct sched_dl_entity		dl;
Linus Torvalds's avatar
Linus Torvalds committed
651

652
#ifdef CONFIG_PREEMPT_NOTIFIERS
653
654
	/* List of struct preempt_notifier: */
	struct hlist_head		preempt_notifiers;
655
656
#endif

657
#ifdef CONFIG_BLK_DEV_IO_TRACE
658
	unsigned int			btrace_seq;
659
#endif
Linus Torvalds's avatar
Linus Torvalds committed
660

661
662
663
	unsigned int			policy;
	int				nr_cpus_allowed;
	cpumask_t			cpus_allowed;
Linus Torvalds's avatar
Linus Torvalds committed
664

Paul E. McKenney's avatar
Paul E. McKenney committed
665
#ifdef CONFIG_PREEMPT_RCU
666
667
668
669
	int				rcu_read_lock_nesting;
	union rcu_special		rcu_read_unlock_special;
	struct list_head		rcu_node_entry;
	struct rcu_node			*rcu_blocked_node;
670
#endif /* #ifdef CONFIG_PREEMPT_RCU */
671

Paul E. McKenney's avatar
Paul E. McKenney committed
672
#ifdef CONFIG_TASKS_RCU
673
	unsigned long			rcu_tasks_nvcsw;
674
675
	u8				rcu_tasks_holdout;
	u8				rcu_tasks_idx;
676
	int				rcu_tasks_idle_cpu;
677
	struct list_head		rcu_tasks_holdout_list;
Paul E. McKenney's avatar
Paul E. McKenney committed
678
#endif /* #ifdef CONFIG_TASKS_RCU */
Paul E. McKenney's avatar
Paul E. McKenney committed
679

680
	struct sched_info		sched_info;
Linus Torvalds's avatar
Linus Torvalds committed
681

682
	struct list_head		tasks;
683
#ifdef CONFIG_SMP
684
685
	struct plist_node		pushable_tasks;
	struct rb_node			pushable_dl_tasks;
686
#endif
Linus Torvalds's avatar
Linus Torvalds committed
687

688
689
	struct mm_struct		*mm;
	struct mm_struct		*active_mm;
690
691

	/* Per-thread vma caching: */
692
	struct vmacache			vmacache;
693

694
695
#ifdef SPLIT_RSS_COUNTING
	struct task_rss_stat		rss_stat;
696
#endif
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
	int				exit_state;
	int				exit_code;
	int				exit_signal;
	/* The signal sent when the parent dies: */
	int				pdeath_signal;
	/* JOBCTL_*, siglock protected: */
	unsigned long			jobctl;

	/* Used for emulating ABI behavior of previous Linux versions: */
	unsigned int			personality;

	/* Scheduler bits, serialized by scheduler locks: */
	unsigned			sched_reset_on_fork:1;
	unsigned			sched_contributes_to_load:1;
	unsigned			sched_migrated:1;
	unsigned			sched_remote_wakeup:1;
	/* Force alignment to the next boundary: */
	unsigned			:0;

	/* Unserialized, strictly 'current' */

	/* Bit to tell LSMs we're in execve(): */
	unsigned			in_execve:1;
	unsigned			in_iowait:1;
#ifndef TIF_RESTORE_SIGMASK
	unsigned			restore_sigmask:1;
723
#endif
724
#ifdef CONFIG_MEMCG
725
	unsigned			memcg_may_oom:1;
726
#ifndef CONFIG_SLOB
727
	unsigned			memcg_kmem_skip_account:1;
728
#endif
729
#endif
730
#ifdef CONFIG_COMPAT_BRK
731
	unsigned			brk_randomized:1;
732
#endif
733
734
735
736
#ifdef CONFIG_CGROUPS
	/* disallow userland-initiated cgroup migration */
	unsigned			no_cgroup_migration:1;
#endif
737

738
	unsigned long			atomic_flags; /* Flags requiring atomic access. */
739

740
	struct restart_block		restart_block;
741

742
743
	pid_t				pid;
	pid_t				tgid;
744

745
#ifdef CONFIG_STACKPROTECTOR
746
747
	/* Canary value for the -fstack-protector GCC feature: */
	unsigned long			stack_canary;
748
#endif
749
	/*
750
	 * Pointers to the (original) parent process, youngest child, younger sibling,
751
	 * older sibling, respectively.  (p->father can be replaced with
Roland McGrath's avatar
Roland McGrath committed
752
	 * p->real_parent->pid)
Linus Torvalds's avatar
Linus Torvalds committed
753
	 */
754
755
756
757
758
759
760

	/* Real parent process: */
	struct task_struct __rcu	*real_parent;

	/* Recipient of SIGCHLD, wait4() reports: */
	struct task_struct __rcu	*parent;

Linus Torvalds's avatar
Linus Torvalds committed
761
	/*
762
	 * Children/sibling form the list of natural children:
Linus Torvalds's avatar
Linus Torvalds committed
763
	 */
764
765
766
	struct list_head		children;
	struct list_head		sibling;
	struct task_struct		*group_leader;
Linus Torvalds's avatar
Linus Torvalds committed
767

Roland McGrath's avatar
Roland McGrath committed
768
	/*
769
770
	 * 'ptraced' is the list of tasks this task is using ptrace() on.
	 *
Roland McGrath's avatar
Roland McGrath committed
771
	 * This includes both natural children and PTRACE_ATTACH targets.
772
	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
Roland McGrath's avatar
Roland McGrath committed
773
	 */
774
775
	struct list_head		ptraced;
	struct list_head		ptrace_entry;
Roland McGrath's avatar
Roland McGrath committed
776

Linus Torvalds's avatar
Linus Torvalds committed
777
	/* PID/PID hash table linkage. */
778
779
780
781
782
	struct pid_link			pids[PIDTYPE_MAX];
	struct list_head		thread_group;
	struct list_head		thread_node;

	struct completion		*vfork_done;
Linus Torvalds's avatar
Linus Torvalds committed
783

784
785
	/* CLONE_CHILD_SETTID: */
	int __user			*set_child_tid;
Linus Torvalds's avatar
Linus Torvalds committed
786

787
788
789
790
791
	/* CLONE_CHILD_CLEARTID: */
	int __user			*clear_child_tid;

	u64				utime;
	u64				stime;
792
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
793
794
	u64				utimescaled;
	u64				stimescaled;
795
#endif
796
797
	u64				gtime;
	struct prev_cputime		prev_cputime;
798
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
799
	struct vtime			vtime;
800
#endif
801
802

#ifdef CONFIG_NO_HZ_FULL
803
	atomic_t			tick_dep_mask;
804
#endif
805
806
807
808
809
810
811
812
813
814
815
816
817
	/* Context switch counts: */
	unsigned long			nvcsw;
	unsigned long			nivcsw;

	/* Monotonic time in nsecs: */
	u64				start_time;

	/* Boot based time in nsecs: */
	u64				real_start_time;

	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
	unsigned long			min_flt;
	unsigned long			maj_flt;
Linus Torvalds's avatar
Linus Torvalds committed
818

819
#ifdef CONFIG_POSIX_TIMERS
820
821
	struct task_cputime		cputime_expires;
	struct list_head		cpu_timers[3];
822
#endif
Linus Torvalds's avatar
Linus Torvalds committed
823

824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
	/* Process credentials: */

	/* Tracer's credentials at attach: */
	const struct cred __rcu		*ptracer_cred;

	/* Objective and real subjective task credentials (COW): */
	const struct cred __rcu		*real_cred;

	/* Effective (overridable) subjective task credentials (COW): */
	const struct cred __rcu		*cred;

	/*
	 * executable name, excluding path.
	 *
	 * - normally initialized setup_new_exec()
	 * - access it with [gs]et_task_comm()
	 * - lock it with task_lock()
	 */
	char				comm[TASK_COMM_LEN];

	struct nameidata		*nameidata;

846
#ifdef CONFIG_SYSVIPC
847
848
	struct sysv_sem			sysvsem;
	struct sysv_shm			sysvshm;
849
#endif
850
#ifdef CONFIG_DETECT_HUNG_TASK
851
	unsigned long			last_switch_count;
852
#endif
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
	/* Filesystem information: */
	struct fs_struct		*fs;

	/* Open file information: */
	struct files_struct		*files;

	/* Namespaces: */
	struct nsproxy			*nsproxy;

	/* Signal handlers: */
	struct signal_struct		*signal;
	struct sighand_struct		*sighand;
	sigset_t			blocked;
	sigset_t			real_blocked;
	/* Restored if set_restore_sigmask() was used: */
	sigset_t			saved_sigmask;
	struct sigpending		pending;
	unsigned long			sas_ss_sp;
	size_t				sas_ss_size;
	unsigned int			sas_ss_flags;

	struct callback_head		*task_works;

	struct audit_context		*audit_context;
Al Viro's avatar
Al Viro committed
877
#ifdef CONFIG_AUDITSYSCALL
878
879
	kuid_t				loginuid;
	unsigned int			sessionid;
Al Viro's avatar
Al Viro committed
880
#endif
881
882
883
884
885
	struct seccomp			seccomp;

	/* Thread group tracking: */
	u32				parent_exec_id;
	u32				self_exec_id;
Linus Torvalds's avatar
Linus Torvalds committed
886

887
888
	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
	spinlock_t			alloc_lock;
Linus Torvalds's avatar
Linus Torvalds committed
889

890
	/* Protection of the PI data structures: */
891
	raw_spinlock_t			pi_lock;
892

893
	struct wake_q_node		wake_q;
894

Ingo Molnar's avatar
Ingo Molnar committed
895
#ifdef CONFIG_RT_MUTEXES
896
	/* PI waiters blocked on a rt_mutex held by this task: */
897
	struct rb_root_cached		pi_waiters;
898
899
	/* Updated under owner's pi_lock and rq lock */
	struct task_struct		*pi_top_task;
900
901
	/* Deadlock detection and priority inheritance handling: */
	struct rt_mutex_waiter		*pi_blocked_on;
Ingo Molnar's avatar
Ingo Molnar committed
902
903
#endif

904
#ifdef CONFIG_DEBUG_MUTEXES
905
906
	/* Mutex deadlock detection: */
	struct mutex_waiter		*blocked_on;
907
#endif
908

909
#ifdef CONFIG_TRACE_IRQFLAGS
910
911
912
913
914
915
916
917
918
919
920
921
922
	unsigned int			irq_events;
	unsigned long			hardirq_enable_ip;
	unsigned long			hardirq_disable_ip;
	unsigned int			hardirq_enable_event;
	unsigned int			hardirq_disable_event;
	int				hardirqs_enabled;
	int				hardirq_context;
	unsigned long			softirq_disable_ip;
	unsigned long			softirq_enable_ip;
	unsigned int			softirq_disable_event;
	unsigned int			softirq_enable_event;
	int				softirqs_enabled;
	int				softirq_context;
923
#endif
924

Ingo Molnar's avatar
Ingo Molnar committed
925
#ifdef CONFIG_LOCKDEP
926
927
928
929
930
# define MAX_LOCK_DEPTH			48UL
	u64				curr_chain_key;
	int				lockdep_depth;
	unsigned int			lockdep_recursion;
	struct held_lock		held_locks[MAX_LOCK_DEPTH];
Ingo Molnar's avatar
Ingo Molnar committed
931
#endif
932

933
#ifdef CONFIG_UBSAN
934
	unsigned int			in_ubsan;
935
#endif
936

937
938
	/* Journalling filesystem info: */
	void				*journal_info;
Linus Torvalds's avatar
Linus Torvalds committed
939

940
941
	/* Stacked block device info: */
	struct bio_list			*bio_list;
942

943
#ifdef CONFIG_BLOCK
944
945
	/* Stack plugging: */
	struct blk_plug			*plug;
946
947
#endif

948
949
950
951
	/* VM state: */
	struct reclaim_state		*reclaim_state;

	struct backing_dev_info		*backing_dev_info;
Linus Torvalds's avatar
Linus Torvalds committed
952

953
	struct io_context		*io_context;
Linus Torvalds's avatar
Linus Torvalds committed
954

955
956
957
	/* Ptrace state: */
	unsigned long			ptrace_message;
	siginfo_t			*last_siginfo;
Linus Torvalds's avatar
Linus Torvalds committed
958

959
960
961
962
963
964
965
966
	struct task_io_accounting	ioac;
#ifdef CONFIG_TASK_XACCT
	/* Accumulated RSS usage: */
	u64				acct_rss_mem1;
	/* Accumulated virtual memory usage: */
	u64				acct_vm_mem1;
	/* stime + utime since last update: */
	u64				acct_timexpd;
Linus Torvalds's avatar
Linus Torvalds committed
967
968
#endif
#ifdef CONFIG_CPUSETS
969
970
971
972
973
974
	/* Protected by ->alloc_lock: */
	nodemask_t			mems_allowed;
	/* Seqence number to catch updates: */
	seqcount_t			mems_allowed_seq;
	int				cpuset_mem_spread_rotor;
	int				cpuset_slab_spread_rotor;
Linus Torvalds's avatar
Linus Torvalds committed
975
#endif
976
#ifdef CONFIG_CGROUPS
977
978
979
980
	/* Control Group info protected by css_set_lock: */
	struct css_set __rcu		*cgroups;
	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
	struct list_head		cg_list;
981
#endif
982
#ifdef CONFIG_INTEL_RDT
983
	u32				closid;
984
	u32				rmid;
Fenghua Yu's avatar
Fenghua Yu committed
985
#endif
986
#ifdef CONFIG_FUTEX
987
	struct robust_list_head __user	*robust_list;
988
989
990
#ifdef CONFIG_COMPAT
	struct compat_robust_list_head __user *compat_robust_list;
#endif
991
992
	struct list_head		pi_state_list;
	struct futex_pi_state		*pi_state_cache;
993
#endif
994
#ifdef CONFIG_PERF_EVENTS
995
996
997
	struct perf_event_context	*perf_event_ctxp[perf_nr_task_contexts];
	struct mutex			perf_event_mutex;
	struct list_head		perf_event_list;
998
#endif
999
#ifdef CONFIG_DEBUG_PREEMPT
1000
	unsigned long			preempt_disable_ip;
1001
#endif
1002
#ifdef CONFIG_NUMA
1003
1004
	/* Protected by alloc_lock: */
	struct mempolicy		*mempolicy;
1005
	short				il_prev;
1006
	short				pref_node_fork;
1007
#endif
1008
#ifdef CONFIG_NUMA_BALANCING
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
	int				numa_scan_seq;
	unsigned int			numa_scan_period;
	unsigned int			numa_scan_period_max;
	int				numa_preferred_nid;
	unsigned long			numa_migrate_retry;
	/* Migration stamp: */
	u64				node_stamp;
	u64				last_task_numa_placement;
	u64				last_sum_exec_runtime;
	struct callback_head		numa_work;

	struct list_head		numa_entry;
	struct numa_group		*numa_group;
1022

1023
	/*
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
	 * numa_faults is an array split into four regions:
	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
	 * in this precise order.
	 *
	 * faults_memory: Exponential decaying average of faults on a per-node
	 * basis. Scheduling placement decisions are made based on these
	 * counts. The values remain static for the duration of a PTE scan.
	 * faults_cpu: Track the nodes the process was running on when a NUMA
	 * hinting fault was incurred.
	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
	 * during the current scan window. When the scan completes, the counts
	 * in faults_memory and faults_cpu decay and these values are copied.
1036
	 */
1037
1038
	unsigned long			*numa_faults;
	unsigned long			total_numa_faults;
1039

1040
1041
	/*
	 * numa_faults_locality tracks if faults recorded during the last
1042
1043
1044
	 * scan window were remote/local or failed to migrate. The task scan
	 * period is adapted based on the locality of the faults with different
	 * weights depending on whether they were shared or private faults
1045
	 */
1046
	unsigned long			numa_faults_locality[3];
1047

1048
	unsigned long			numa_pages_migrated;
1049
1050
#endif /* CONFIG_NUMA_BALANCING */

1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
#ifdef CONFIG_RSEQ
	struct rseq __user *rseq;
	u32 rseq_len;
	u32 rseq_sig;
	/*
	 * RmW on rseq_event_mask must be performed atomically
	 * with respect to preemption.
	 */
	unsigned long rseq_event_mask;
#endif

1062
	struct tlbflush_unmap_batch	tlb_ubc;
1063

1064
	struct rcu_head			rcu;
1065

1066
1067
	/* Cache last used pipe for splice(): */
	struct pipe_inode_info		*splice_pipe;
1068

1069
	struct page_frag		task_frag;
1070

1071
1072
#ifdef CONFIG_TASK_DELAY_ACCT
	struct task_delay_info		*delays;
1073
#endif
1074

1075
#ifdef CONFIG_FAULT_INJECTION
1076
	int				make_it_fail;
1077
	unsigned int			fail_nth;
1078
#endif
1079
	/*
1080
1081
	 * When (nr_dirtied >= nr_dirtied_pause), it's time to call
	 * balance_dirty_pages() for a dirty throttling pause:
1082
	 */
1083
1084
1085