sched.h 53 KB
Newer Older
1
/* SPDX-License-Identifier: GPL-2.0 */
Linus Torvalds's avatar
Linus Torvalds committed
2
3
4
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

5
6
7
8
/*
 * Define 'struct task_struct' and provide the main scheduler
 * APIs (schedule(), wakeup variants, etc.)
 */
9

10
#include <uapi/linux/sched.h>
11

12
#include <asm/current.h>
Linus Torvalds's avatar
Linus Torvalds committed
13

14
#include <linux/pid.h>
Linus Torvalds's avatar
Linus Torvalds committed
15
#include <linux/sem.h>
16
#include <linux/shm.h>
17
18
19
20
#include <linux/kcov.h>
#include <linux/mutex.h>
#include <linux/plist.h>
#include <linux/hrtimer.h>
Linus Torvalds's avatar
Linus Torvalds committed
21
#include <linux/seccomp.h>
22
#include <linux/nodemask.h>
23
#include <linux/rcupdate.h>
24
#include <linux/refcount.h>
25
#include <linux/resource.h>
Arjan van de Ven's avatar
Arjan van de Ven committed
26
#include <linux/latencytop.h>
27
28
#include <linux/sched/prio.h>
#include <linux/signal_types.h>
29
#include <linux/psi_types.h>
30
31
#include <linux/mm_types_task.h>
#include <linux/task_io_accounting.h>
32
#include <linux/rseq.h>
33

34
/* task_struct member predeclarations (sorted alphabetically): */
35
36
struct audit_context;
struct backing_dev_info;
37
struct bio_list;
38
struct blk_plug;
39
40
41
42
43
struct cfs_rq;
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct mempolicy;
44
struct nameidata;
45
46
47
48
49
50
struct nsproxy;
struct perf_event_context;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
51
struct capture_control;
52
53
54
struct robust_list_head;
struct sched_attr;
struct sched_param;
Ingo Molnar's avatar
Ingo Molnar committed
55
struct seq_file;
56
57
58
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
59
struct task_group;
Linus Torvalds's avatar
Linus Torvalds committed
60

61
62
63
64
65
66
67
68
69
70
/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */
71
72

/* Used in tsk->state: */
73
74
75
76
77
#define TASK_RUNNING			0x0000
#define TASK_INTERRUPTIBLE		0x0001
#define TASK_UNINTERRUPTIBLE		0x0002
#define __TASK_STOPPED			0x0004
#define __TASK_TRACED			0x0008
78
/* Used in tsk->exit_state: */
79
80
#define EXIT_DEAD			0x0010
#define EXIT_ZOMBIE			0x0020
81
82
#define EXIT_TRACE			(EXIT_ZOMBIE | EXIT_DEAD)
/* Used in tsk->state again: */
83
84
85
86
#define TASK_PARKED			0x0040
#define TASK_DEAD			0x0080
#define TASK_WAKEKILL			0x0100
#define TASK_WAKING			0x0200
87
88
89
#define TASK_NOLOAD			0x0400
#define TASK_NEW			0x0800
#define TASK_STATE_MAX			0x1000
90
91
92
93
94
95
96
97
98
99
100
101
102
103

/* Convenience macros for the sake of set_current_state: */
#define TASK_KILLABLE			(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED			(TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED			(TASK_WAKEKILL | __TASK_TRACED)

#define TASK_IDLE			(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

/* Convenience macros for the sake of wake_up(): */
#define TASK_NORMAL			(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)

/* get_task_state(): */
#define TASK_REPORT			(TASK_RUNNING | TASK_INTERRUPTIBLE | \
					 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
104
105
					 __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
					 TASK_PARKED)
106
107
108
109
110
111
112
113
114
115

#define task_is_traced(task)		((task->state & __TASK_TRACED) != 0)

#define task_is_stopped(task)		((task->state & __TASK_STOPPED) != 0)

#define task_is_stopped_or_traced(task)	((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)

#define task_contributes_to_load(task)	((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
					 (task->flags & PF_FROZEN) == 0 && \
					 (task->state & TASK_NOLOAD) == 0)
Linus Torvalds's avatar
Linus Torvalds committed
116

Peter Zijlstra's avatar
Peter Zijlstra committed
117
118
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

119
120
121
122
123
/*
 * Special states are those that do not use the normal wait-loop pattern. See
 * the comment with set_special_state().
 */
#define is_special_task_state(state)				\
124
	((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
125

Peter Zijlstra's avatar
Peter Zijlstra committed
126
127
#define __set_current_state(state_value)			\
	do {							\
128
		WARN_ON_ONCE(is_special_task_state(state_value));\
Peter Zijlstra's avatar
Peter Zijlstra committed
129
130
131
		current->task_state_change = _THIS_IP_;		\
		current->state = (state_value);			\
	} while (0)
132

Peter Zijlstra's avatar
Peter Zijlstra committed
133
134
#define set_current_state(state_value)				\
	do {							\
135
		WARN_ON_ONCE(is_special_task_state(state_value));\
Peter Zijlstra's avatar
Peter Zijlstra committed
136
		current->task_state_change = _THIS_IP_;		\
137
		smp_store_mb(current->state, (state_value));	\
Peter Zijlstra's avatar
Peter Zijlstra committed
138
139
	} while (0)

140
141
142
143
144
145
146
147
148
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		WARN_ON_ONCE(!is_special_task_state(state_value));	\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->task_state_change = _THIS_IP_;			\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)
Peter Zijlstra's avatar
Peter Zijlstra committed
149
#else
150
151
152
153
154
/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
155
 *   for (;;) {
156
 *	set_current_state(TASK_UNINTERRUPTIBLE);
157
158
159
160
161
162
163
164
165
166
167
168
169
 *	if (!need_sleep)
 *		break;
 *
 *	schedule();
 *   }
 *   __set_current_state(TASK_RUNNING);
 *
 * If the caller does not need such serialisation (because, for instance, the
 * condition test and condition change and wakeup are under the same lock) then
 * use __set_current_state().
 *
 * The above is typically ordered against the wakeup, which does:
 *
170
171
 *   need_sleep = false;
 *   wake_up_state(p, TASK_UNINTERRUPTIBLE);
172
 *
173
174
 * where wake_up_state() executes a full memory barrier before accessing the
 * task state.
175
176
177
178
 *
 * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
179
 *
180
 * However, with slightly different timing the wakeup TASK_RUNNING store can
181
 * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
182
183
 * a problem either because that will result in one extra go around the loop
 * and our @cond test will save the day.
184
 *
185
 * Also see the comments of try_to_wake_up().
186
 */
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
#define __set_current_state(state_value)				\
	current->state = (state_value)

#define set_current_state(state_value)					\
	smp_store_mb(current->state, (state_value))

/*
 * set_special_state() should be used for those states when the blocking task
 * can not use the regular condition based wait-loop. In that case we must
 * serialize against wakeups such that any possible in-flight TASK_RUNNING stores
 * will not collide with our state change.
 */
#define set_special_state(state_value)					\
	do {								\
		unsigned long flags; /* may shadow */			\
		raw_spin_lock_irqsave(&current->pi_lock, flags);	\
		current->state = (state_value);				\
		raw_spin_unlock_irqrestore(&current->pi_lock, flags);	\
	} while (0)

Peter Zijlstra's avatar
Peter Zijlstra committed
207
208
#endif

209
210
/* Task command name length: */
#define TASK_COMM_LEN			16
Linus Torvalds's avatar
Linus Torvalds committed
211
212
213

extern void scheduler_tick(void);

214
215
216
217
218
219
220
#define	MAX_SCHEDULE_TIMEOUT		LONG_MAX

extern long schedule_timeout(long timeout);
extern long schedule_timeout_interruptible(long timeout);
extern long schedule_timeout_killable(long timeout);
extern long schedule_timeout_uninterruptible(long timeout);
extern long schedule_timeout_idle(long timeout);
Linus Torvalds's avatar
Linus Torvalds committed
221
asmlinkage void schedule(void);
222
extern void schedule_preempt_disabled(void);
Linus Torvalds's avatar
Linus Torvalds committed
223

224
225
extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
226
extern long io_schedule_timeout(long timeout);
227
extern void io_schedule(void);
228

229
/**
230
 * struct prev_cputime - snapshot of system and user cputime
231
232
 * @utime: time spent in user mode
 * @stime: time spent in system mode
233
 * @lock: protects the above two fields
234
 *
235
236
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
237
 */
238
239
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
240
241
242
	u64				utime;
	u64				stime;
	raw_spinlock_t			lock;
243
#endif
244
245
};

246
247
/**
 * struct task_cputime - collected CPU time counts
248
249
 * @utime:		time spent in user mode, in nanoseconds
 * @stime:		time spent in kernel mode, in nanoseconds
250
 * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
251
 *
252
253
254
 * This structure groups together three kinds of CPU time that are tracked for
 * threads and thread groups.  Most things considering CPU time want to group
 * these counts together and treat all three of them in parallel.
255
256
 */
struct task_cputime {
257
258
259
	u64				utime;
	u64				stime;
	unsigned long long		sum_exec_runtime;
260
};
261

262
263
264
265
/* Alternate field names when used on cache expirations: */
#define virt_exp			utime
#define prof_exp			stime
#define sched_exp			sum_exec_runtime
266

267
268
269
270
271
272
273
274
275
276
277
278
279
enum vtime_state {
	/* Task is sleeping or running in a CPU with VTIME inactive: */
	VTIME_INACTIVE = 0,
	/* Task runs in userspace in a CPU with VTIME active: */
	VTIME_USER,
	/* Task runs in kernelspace in a CPU with VTIME active: */
	VTIME_SYS,
};

struct vtime {
	seqcount_t		seqcount;
	unsigned long long	starttime;
	enum vtime_state	state;
280
281
282
	u64			utime;
	u64			stime;
	u64			gtime;
283
284
};

Linus Torvalds's avatar
Linus Torvalds committed
285
struct sched_info {
286
#ifdef CONFIG_SCHED_INFO
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
	/* Cumulative counters: */

	/* # of times we have run on this CPU: */
	unsigned long			pcount;

	/* Time spent waiting on a runqueue: */
	unsigned long long		run_delay;

	/* Timestamps: */

	/* When did we last run on a CPU? */
	unsigned long long		last_arrival;

	/* When were we last queued to run? */
	unsigned long long		last_queued;
Linus Torvalds's avatar
Linus Torvalds committed
302

303
#endif /* CONFIG_SCHED_INFO */
304
};
Linus Torvalds's avatar
Linus Torvalds committed
305

306
307
308
309
310
311
312
/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
313
314
# define SCHED_FIXEDPOINT_SHIFT		10
# define SCHED_FIXEDPOINT_SCALE		(1L << SCHED_FIXEDPOINT_SHIFT)
315

Ingo Molnar's avatar
Ingo Molnar committed
316
struct load_weight {
317
318
	unsigned long			weight;
	u32				inv_weight;
Ingo Molnar's avatar
Ingo Molnar committed
319
320
};

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
/**
 * struct util_est - Estimation utilization of FAIR tasks
 * @enqueued: instantaneous estimated utilization of a task/cpu
 * @ewma:     the Exponential Weighted Moving Average (EWMA)
 *            utilization of a task
 *
 * Support data structure to track an Exponential Weighted Moving Average
 * (EWMA) of a FAIR task's utilization. New samples are added to the moving
 * average each time a task completes an activation. Sample's weight is chosen
 * so that the EWMA will be relatively insensitive to transient changes to the
 * task's workload.
 *
 * The enqueued attribute has a slightly different meaning for tasks and cpus:
 * - task:   the task's util_avg at last task dequeue time
 * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
 * Thus, the util_est.enqueued of a task represents the contribution on the
 * estimated utilization of the CPU where that task is currently enqueued.
 *
 * Only for tasks we track a moving average of the past instantaneous
 * estimated utilization. This allows to absorb sporadic drops in utilization
 * of an otherwise almost periodic task.
 */
struct util_est {
	unsigned int			enqueued;
	unsigned int			ewma;
#define UTIL_EST_WEIGHT_SHIFT		2
347
} __attribute__((__aligned__(sizeof(u64))));
348

349
/*
350
351
352
353
354
355
356
357
358
 * The load_avg/util_avg accumulates an infinite geometric series
 * (see __update_load_avg() in kernel/sched/fair.c).
 *
 * [load_avg definition]
 *
 *   load_avg = runnable% * scale_load_down(load)
 *
 * where runnable% is the time ratio that a sched_entity is runnable.
 * For cfs_rq, it is the aggregated load_avg of all runnable and
359
 * blocked sched_entities.
360
361
362
363
364
365
366
367
368
 *
 * [util_avg definition]
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE
 *
 * where running% is the time ratio that a sched_entity is running on
 * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
 * and blocked sched_entities.
 *
369
370
371
 * load_avg and util_avg don't direcly factor frequency scaling and CPU
 * capacity scaling. The scaling is done through the rq_clock_pelt that
 * is used for computing those signals (see update_rq_clock_pelt())
372
 *
373
374
375
376
 * N.B., the above ratios (runnable% and running%) themselves are in the
 * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
 * to as large a range as necessary. This is for example reflected by
 * util_avg's SCHED_CAPACITY_SCALE.
377
378
379
380
381
382
383
384
385
386
387
388
389
390
 *
 * [Overflow issue]
 *
 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
 * with the highest load (=88761), always runnable on a single cfs_rq,
 * and should not overflow as the number already hits PID_MAX_LIMIT.
 *
 * For all other cases (including 32-bit kernels), struct load_weight's
 * weight will overflow first before we do, because:
 *
 *    Max(load_avg) <= Max(load.weight)
 *
 * Then it is the load_weight's responsibility to consider overflow
 * issues.
391
 */
392
struct sched_avg {
393
394
	u64				last_update_time;
	u64				load_sum;
395
	u64				runnable_load_sum;
396
397
398
	u32				util_sum;
	u32				period_contrib;
	unsigned long			load_avg;
399
	unsigned long			runnable_load_avg;
400
	unsigned long			util_avg;
401
	struct util_est			util_est;
402
} ____cacheline_aligned;
403

404
struct sched_statistics {
405
#ifdef CONFIG_SCHEDSTATS
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
	u64				wait_start;
	u64				wait_max;
	u64				wait_count;
	u64				wait_sum;
	u64				iowait_count;
	u64				iowait_sum;

	u64				sleep_start;
	u64				sleep_max;
	s64				sum_sleep_runtime;

	u64				block_start;
	u64				block_max;
	u64				exec_max;
	u64				slice_max;

	u64				nr_migrations_cold;
	u64				nr_failed_migrations_affine;
	u64				nr_failed_migrations_running;
	u64				nr_failed_migrations_hot;
	u64				nr_forced_migrations;

	u64				nr_wakeups;
	u64				nr_wakeups_sync;
	u64				nr_wakeups_migrate;
	u64				nr_wakeups_local;
	u64				nr_wakeups_remote;
	u64				nr_wakeups_affine;
	u64				nr_wakeups_affine_attempts;
	u64				nr_wakeups_passive;
	u64				nr_wakeups_idle;
437
#endif
438
};
439
440

struct sched_entity {
441
442
	/* For load-balancing: */
	struct load_weight		load;
443
	unsigned long			runnable_weight;
444
445
446
	struct rb_node			run_node;
	struct list_head		group_node;
	unsigned int			on_rq;
447

448
449
450
451
	u64				exec_start;
	u64				sum_exec_runtime;
	u64				vruntime;
	u64				prev_sum_exec_runtime;
452

453
	u64				nr_migrations;
454

455
	struct sched_statistics		statistics;
456

Ingo Molnar's avatar
Ingo Molnar committed
457
#ifdef CONFIG_FAIR_GROUP_SCHED
458
459
	int				depth;
	struct sched_entity		*parent;
Ingo Molnar's avatar
Ingo Molnar committed
460
	/* rq on which this entity is (to be) queued: */
461
	struct cfs_rq			*cfs_rq;
Ingo Molnar's avatar
Ingo Molnar committed
462
	/* rq "owned" by this entity/group: */
463
	struct cfs_rq			*my_q;
Ingo Molnar's avatar
Ingo Molnar committed
464
#endif
465

466
#ifdef CONFIG_SMP
467
468
469
470
471
472
	/*
	 * Per entity load average tracking.
	 *
	 * Put into separate cache line so it does not
	 * collide with read-mostly values above.
	 */
473
	struct sched_avg		avg;
474
#endif
Ingo Molnar's avatar
Ingo Molnar committed
475
};
476

Peter Zijlstra's avatar
Peter Zijlstra committed
477
struct sched_rt_entity {
478
479
480
481
482
483
484
485
	struct list_head		run_list;
	unsigned long			timeout;
	unsigned long			watchdog_stamp;
	unsigned int			time_slice;
	unsigned short			on_rq;
	unsigned short			on_list;

	struct sched_rt_entity		*back;
486
#ifdef CONFIG_RT_GROUP_SCHED
487
	struct sched_rt_entity		*parent;
Peter Zijlstra's avatar
Peter Zijlstra committed
488
	/* rq on which this entity is (to be) queued: */
489
	struct rt_rq			*rt_rq;
Peter Zijlstra's avatar
Peter Zijlstra committed
490
	/* rq "owned" by this entity/group: */
491
	struct rt_rq			*my_q;
Peter Zijlstra's avatar
Peter Zijlstra committed
492
#endif
493
} __randomize_layout;
Peter Zijlstra's avatar
Peter Zijlstra committed
494

495
struct sched_dl_entity {
496
	struct rb_node			rb_node;
497
498
499

	/*
	 * Original scheduling parameters. Copied here from sched_attr
500
501
	 * during sched_setattr(), they will remain the same until
	 * the next sched_setattr().
502
	 */
503
504
505
	u64				dl_runtime;	/* Maximum runtime for each instance	*/
	u64				dl_deadline;	/* Relative deadline of each instance	*/
	u64				dl_period;	/* Separation of two instances (period) */
506
	u64				dl_bw;		/* dl_runtime / dl_period		*/
507
	u64				dl_density;	/* dl_runtime / dl_deadline		*/
508
509
510

	/*
	 * Actual scheduling parameters. Initialized with the values above,
511
	 * they are continuously updated during task execution. Note that
512
513
	 * the remaining runtime could be < 0 in case we are in overrun.
	 */
514
515
516
	s64				runtime;	/* Remaining runtime for this instance	*/
	u64				deadline;	/* Absolute deadline for this instance	*/
	unsigned int			flags;		/* Specifying the scheduler behaviour	*/
517
518
519
520
521
522
523
524

	/*
	 * Some bool flags:
	 *
	 * @dl_throttled tells if we exhausted the runtime. If so, the
	 * task has to wait for a replenishment to be performed at the
	 * next firing of dl_timer.
	 *
525
526
	 * @dl_boosted tells if we are boosted due to DI. If so we are
	 * outside bandwidth enforcement mechanism (but only until we
527
528
	 * exit the critical section);
	 *
529
	 * @dl_yielded tells if task gave up the CPU before consuming
530
	 * all its available runtime during the last job.
531
532
533
534
535
536
537
	 *
	 * @dl_non_contending tells if the task is inactive while still
	 * contributing to the active utilization. In other words, it
	 * indicates if the inactive timer has been armed and its handler
	 * has not been executed yet. This flag is useful to avoid race
	 * conditions between the inactive timer handler and the wakeup
	 * code.
538
539
540
	 *
	 * @dl_overrun tells if the task asked to be informed about runtime
	 * overruns.
541
	 */
542
543
544
545
	unsigned int			dl_throttled      : 1;
	unsigned int			dl_boosted        : 1;
	unsigned int			dl_yielded        : 1;
	unsigned int			dl_non_contending : 1;
546
	unsigned int			dl_overrun	  : 1;
547
548
549
550
551

	/*
	 * Bandwidth enforcement timer. Each -deadline task has its
	 * own bandwidth to be enforced, thus we need one timer per task.
	 */
552
	struct hrtimer			dl_timer;
553
554
555
556
557
558
559
560
561

	/*
	 * Inactive timer, responsible for decreasing the active utilization
	 * at the "0-lag time". When a -deadline task blocks, it contributes
	 * to GRUB's active utilization until the "0-lag time", hence a
	 * timer is needed to decrease the active utilization at the correct
	 * time.
	 */
	struct hrtimer inactive_timer;
562
};
563

564
565
union rcu_special {
	struct {
566
567
		u8			blocked;
		u8			need_qs;
568
569
		u8			exp_hint; /* Hint for performance. */
		u8			pad; /* No garbage from compiler! */
570
	} b; /* Bits. */
571
	u32 s; /* Set of bits. */
572
};
573

Peter Zijlstra's avatar
Peter Zijlstra committed
574
575
576
enum perf_event_task_context {
	perf_invalid_context = -1,
	perf_hw_context = 0,
577
	perf_sw_context,
Peter Zijlstra's avatar
Peter Zijlstra committed
578
579
580
	perf_nr_task_contexts,
};

581
582
583
584
struct wake_q_node {
	struct wake_q_node *next;
};

Linus Torvalds's avatar
Linus Torvalds committed
585
struct task_struct {
586
587
588
589
590
#ifdef CONFIG_THREAD_INFO_IN_TASK
	/*
	 * For reasons of header soup (see current_thread_info()), this
	 * must be the first element of task_struct.
	 */
591
	struct thread_info		thread_info;
592
#endif
593
594
	/* -1 unrunnable, 0 runnable, >0 stopped: */
	volatile long			state;
595
596
597
598
599
600
601

	/*
	 * This begins the randomizable portion of task_struct. Only
	 * scheduling-critical items should be added above here.
	 */
	randomized_struct_fields_start

602
	void				*stack;
603
	refcount_t			usage;
604
605
606
	/* Per task flags (PF_*), defined further below: */
	unsigned int			flags;
	unsigned int			ptrace;
Linus Torvalds's avatar
Linus Torvalds committed
607

608
#ifdef CONFIG_SMP
609
610
	struct llist_node		wake_entry;
	int				on_cpu;
611
#ifdef CONFIG_THREAD_INFO_IN_TASK
612
613
	/* Current CPU: */
	unsigned int			cpu;
614
#endif
615
616
617
	unsigned int			wakee_flips;
	unsigned long			wakee_flip_decay_ts;
	struct task_struct		*last_wakee;
618

619
620
621
622
623
624
625
626
	/*
	 * recent_used_cpu is initially set as the last CPU used by a task
	 * that wakes affine another task. Waker/wakee relationships can
	 * push tasks around a CPU where each wakeup moves to the next one.
	 * Tracking a recently used CPU allows a quick search for a recently
	 * used CPU that may be idle.
	 */
	int				recent_used_cpu;
627
	int				wake_cpu;
628
#endif
629
630
631
632
633
634
	int				on_rq;

	int				prio;
	int				static_prio;
	int				normal_prio;
	unsigned int			rt_priority;
635

636
637
638
	const struct sched_class	*sched_class;
	struct sched_entity		se;
	struct sched_rt_entity		rt;
639
#ifdef CONFIG_CGROUP_SCHED
640
	struct task_group		*sched_task_group;
641
#endif
642
	struct sched_dl_entity		dl;
Linus Torvalds's avatar
Linus Torvalds committed
643

644
#ifdef CONFIG_PREEMPT_NOTIFIERS
645
646
	/* List of struct preempt_notifier: */
	struct hlist_head		preempt_notifiers;
647
648
#endif

649
#ifdef CONFIG_BLK_DEV_IO_TRACE
650
	unsigned int			btrace_seq;
651
#endif
Linus Torvalds's avatar
Linus Torvalds committed
652

653
654
655
	unsigned int			policy;
	int				nr_cpus_allowed;
	cpumask_t			cpus_allowed;
Linus Torvalds's avatar
Linus Torvalds committed
656

Paul E. McKenney's avatar
Paul E. McKenney committed
657
#ifdef CONFIG_PREEMPT_RCU
658
659
660
661
	int				rcu_read_lock_nesting;
	union rcu_special		rcu_read_unlock_special;
	struct list_head		rcu_node_entry;
	struct rcu_node			*rcu_blocked_node;
662
#endif /* #ifdef CONFIG_PREEMPT_RCU */
663

Paul E. McKenney's avatar
Paul E. McKenney committed
664
#ifdef CONFIG_TASKS_RCU
665
	unsigned long			rcu_tasks_nvcsw;
666
667
	u8				rcu_tasks_holdout;
	u8				rcu_tasks_idx;
668
	int				rcu_tasks_idle_cpu;
669
	struct list_head		rcu_tasks_holdout_list;
Paul E. McKenney's avatar
Paul E. McKenney committed
670
#endif /* #ifdef CONFIG_TASKS_RCU */
Paul E. McKenney's avatar
Paul E. McKenney committed
671

672
	struct sched_info		sched_info;
Linus Torvalds's avatar
Linus Torvalds committed
673

674
	struct list_head		tasks;
675
#ifdef CONFIG_SMP
676
677
	struct plist_node		pushable_tasks;
	struct rb_node			pushable_dl_tasks;
678
#endif
Linus Torvalds's avatar
Linus Torvalds committed
679

680
681
	struct mm_struct		*mm;
	struct mm_struct		*active_mm;
682
683

	/* Per-thread vma caching: */
684
	struct vmacache			vmacache;
685

686
687
#ifdef SPLIT_RSS_COUNTING
	struct task_rss_stat		rss_stat;
688
#endif
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
	int				exit_state;
	int				exit_code;
	int				exit_signal;
	/* The signal sent when the parent dies: */
	int				pdeath_signal;
	/* JOBCTL_*, siglock protected: */
	unsigned long			jobctl;

	/* Used for emulating ABI behavior of previous Linux versions: */
	unsigned int			personality;

	/* Scheduler bits, serialized by scheduler locks: */
	unsigned			sched_reset_on_fork:1;
	unsigned			sched_contributes_to_load:1;
	unsigned			sched_migrated:1;
	unsigned			sched_remote_wakeup:1;
705
706
707
708
#ifdef CONFIG_PSI
	unsigned			sched_psi_wake_requeue:1;
#endif

709
710
711
712
713
714
715
716
717
718
	/* Force alignment to the next boundary: */
	unsigned			:0;

	/* Unserialized, strictly 'current' */

	/* Bit to tell LSMs we're in execve(): */
	unsigned			in_execve:1;
	unsigned			in_iowait:1;
#ifndef TIF_RESTORE_SIGMASK
	unsigned			restore_sigmask:1;
719
#endif
720
#ifdef CONFIG_MEMCG
721
	unsigned			in_user_fault:1;
722
#endif
723
#ifdef CONFIG_COMPAT_BRK
724
	unsigned			brk_randomized:1;
725
#endif
726
727
728
#ifdef CONFIG_CGROUPS
	/* disallow userland-initiated cgroup migration */
	unsigned			no_cgroup_migration:1;
Roman Gushchin's avatar
Roman Gushchin committed
729
730
	/* task is frozen/stopped (used by the cgroup freezer) */
	unsigned			frozen:1;
731
#endif
732
733
734
735
#ifdef CONFIG_BLK_CGROUP
	/* to be used once the psi infrastructure lands upstream. */
	unsigned			use_memdelay:1;
#endif
736

737
	unsigned long			atomic_flags; /* Flags requiring atomic access. */
738

739
	struct restart_block		restart_block;
740

741
742
	pid_t				pid;
	pid_t				tgid;
743

744
#ifdef CONFIG_STACKPROTECTOR
745
746
	/* Canary value for the -fstack-protector GCC feature: */
	unsigned long			stack_canary;
747
#endif
748
	/*
749
	 * Pointers to the (original) parent process, youngest child, younger sibling,
750
	 * older sibling, respectively.  (p->father can be replaced with
Roland McGrath's avatar
Roland McGrath committed
751
	 * p->real_parent->pid)
Linus Torvalds's avatar
Linus Torvalds committed
752
	 */
753
754
755
756
757
758
759

	/* Real parent process: */
	struct task_struct __rcu	*real_parent;

	/* Recipient of SIGCHLD, wait4() reports: */
	struct task_struct __rcu	*parent;

Linus Torvalds's avatar
Linus Torvalds committed
760
	/*
761
	 * Children/sibling form the list of natural children:
Linus Torvalds's avatar
Linus Torvalds committed
762
	 */
763
764
765
	struct list_head		children;
	struct list_head		sibling;
	struct task_struct		*group_leader;
Linus Torvalds's avatar
Linus Torvalds committed
766

Roland McGrath's avatar
Roland McGrath committed
767
	/*
768
769
	 * 'ptraced' is the list of tasks this task is using ptrace() on.
	 *
Roland McGrath's avatar
Roland McGrath committed
770
	 * This includes both natural children and PTRACE_ATTACH targets.
771
	 * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
Roland McGrath's avatar
Roland McGrath committed
772
	 */
773
774
	struct list_head		ptraced;
	struct list_head		ptrace_entry;
Roland McGrath's avatar
Roland McGrath committed
775

Linus Torvalds's avatar
Linus Torvalds committed
776
	/* PID/PID hash table linkage. */
777
778
	struct pid			*thread_pid;
	struct hlist_node		pid_links[PIDTYPE_MAX];
779
780
781
782
	struct list_head		thread_group;
	struct list_head		thread_node;

	struct completion		*vfork_done;
Linus Torvalds's avatar
Linus Torvalds committed
783

784
785
	/* CLONE_CHILD_SETTID: */
	int __user			*set_child_tid;
Linus Torvalds's avatar
Linus Torvalds committed
786

787
788
789
790
791
	/* CLONE_CHILD_CLEARTID: */
	int __user			*clear_child_tid;

	u64				utime;
	u64				stime;
792
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
793
794
	u64				utimescaled;
	u64				stimescaled;
795
#endif
796
797
	u64				gtime;
	struct prev_cputime		prev_cputime;
798
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
799
	struct vtime			vtime;
800
#endif
801
802

#ifdef CONFIG_NO_HZ_FULL
803
	atomic_t			tick_dep_mask;
804
#endif
805
806
807
808
809
810
811
812
813
814
815
816
817
	/* Context switch counts: */
	unsigned long			nvcsw;
	unsigned long			nivcsw;

	/* Monotonic time in nsecs: */
	u64				start_time;

	/* Boot based time in nsecs: */
	u64				real_start_time;

	/* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
	unsigned long			min_flt;
	unsigned long			maj_flt;
Linus Torvalds's avatar
Linus Torvalds committed
818

819
#ifdef CONFIG_POSIX_TIMERS
820
821
	struct task_cputime		cputime_expires;
	struct list_head		cpu_timers[3];
822
#endif
Linus Torvalds's avatar
Linus Torvalds committed
823

824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
	/* Process credentials: */

	/* Tracer's credentials at attach: */
	const struct cred __rcu		*ptracer_cred;

	/* Objective and real subjective task credentials (COW): */
	const struct cred __rcu		*real_cred;

	/* Effective (overridable) subjective task credentials (COW): */
	const struct cred __rcu		*cred;

	/*
	 * executable name, excluding path.
	 *
	 * - normally initialized setup_new_exec()
	 * - access it with [gs]et_task_comm()
	 * - lock it with task_lock()
	 */
	char				comm[TASK_COMM_LEN];

	struct nameidata		*nameidata;

846
#ifdef CONFIG_SYSVIPC
847
848
	struct sysv_sem			sysvsem;
	struct sysv_shm			sysvshm;
849
#endif
850
#ifdef CONFIG_DETECT_HUNG_TASK
851
	unsigned long			last_switch_count;
852
	unsigned long			last_switch_time;
853
#endif
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
	/* Filesystem information: */
	struct fs_struct		*fs;

	/* Open file information: */
	struct files_struct		*files;

	/* Namespaces: */
	struct nsproxy			*nsproxy;

	/* Signal handlers: */
	struct signal_struct		*signal;
	struct sighand_struct		*sighand;
	sigset_t			blocked;
	sigset_t			real_blocked;
	/* Restored if set_restore_sigmask() was used: */
	sigset_t			saved_sigmask;
	struct sigpending		pending;
	unsigned long			sas_ss_sp;
	size_t				sas_ss_size;
	unsigned int			sas_ss_flags;

	struct callback_head		*task_works;

877
#ifdef CONFIG_AUDIT
Al Viro's avatar
Al Viro committed
878
#ifdef CONFIG_AUDITSYSCALL
879
880
	struct audit_context		*audit_context;
#endif
881
882
	kuid_t				loginuid;
	unsigned int			sessionid;
Al Viro's avatar
Al Viro committed
883
#endif
884
885
886
887
888
	struct seccomp			seccomp;

	/* Thread group tracking: */
	u32				parent_exec_id;
	u32				self_exec_id;
Linus Torvalds's avatar
Linus Torvalds committed
889

890
891
	/* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
	spinlock_t			alloc_lock;
Linus Torvalds's avatar
Linus Torvalds committed
892

893
	/* Protection of the PI data structures: */
894
	raw_spinlock_t			pi_lock;
895

896
	struct wake_q_node		wake_q;
897

Ingo Molnar's avatar
Ingo Molnar committed
898
#ifdef CONFIG_RT_MUTEXES
899
	/* PI waiters blocked on a rt_mutex held by this task: */
900
	struct rb_root_cached		pi_waiters;
901
902
	/* Updated under owner's pi_lock and rq lock */
	struct task_struct		*pi_top_task;
903
904
	/* Deadlock detection and priority inheritance handling: */
	struct rt_mutex_waiter		*pi_blocked_on;
Ingo Molnar's avatar
Ingo Molnar committed
905
906
#endif

907
#ifdef CONFIG_DEBUG_MUTEXES
908
909
	/* Mutex deadlock detection: */
	struct mutex_waiter		*blocked_on;
910
#endif
911

912
#ifdef CONFIG_TRACE_IRQFLAGS
913
914
915
916
917
918
919
920
921
922
923
924
925
	unsigned int			irq_events;
	unsigned long			hardirq_enable_ip;
	unsigned long			hardirq_disable_ip;
	unsigned int			hardirq_enable_event;
	unsigned int			hardirq_disable_event;
	int				hardirqs_enabled;
	int				hardirq_context;
	unsigned long			softirq_disable_ip;
	unsigned long			softirq_enable_ip;
	unsigned int			softirq_disable_event;
	unsigned int			softirq_enable_event;
	int				softirqs_enabled;
	int				softirq_context;
926
#endif
927

Ingo Molnar's avatar
Ingo Molnar committed
928
#ifdef CONFIG_LOCKDEP
929
930
931
932
933
# define MAX_LOCK_DEPTH			48UL
	u64				curr_chain_key;
	int				lockdep_depth;
	unsigned int			lockdep_recursion;
	struct held_lock		held_locks[MAX_LOCK_DEPTH];
Ingo Molnar's avatar
Ingo Molnar committed
934
#endif
935

936
#ifdef CONFIG_UBSAN
937
	unsigned int			in_ubsan;
938
#endif
939

940
941
	/* Journalling filesystem info: */
	void				*journal_info;
Linus Torvalds's avatar
Linus Torvalds committed
942

943
944
	/* Stacked block device info: */
	struct bio_list			*bio_list;
945

946
#ifdef CONFIG_BLOCK
947
948
	/* Stack plugging: */
	struct blk_plug			*plug;
949
950
#endif

951
952
953
954
	/* VM state: */
	struct reclaim_state		*reclaim_state;

	struct backing_dev_info		*backing_dev_info;
Linus Torvalds's avatar
Linus Torvalds committed
955

956
	struct io_context		*io_context;
Linus Torvalds's avatar
Linus Torvalds committed
957

958
959
960
#ifdef CONFIG_COMPACTION
	struct capture_control		*capture_control;
#endif
961
962
	/* Ptrace state: */
	unsigned long			ptrace_message;
963
	kernel_siginfo_t		*last_siginfo;
Linus Torvalds's avatar
Linus Torvalds committed
964

965
	struct task_io_accounting	ioac;
966
967
968
969
#ifdef CONFIG_PSI
	/* Pressure stall state */
	unsigned int			psi_flags;
#endif
970
971
972
973
974
975
976
#ifdef CONFIG_TASK_XACCT
	/* Accumulated RSS usage: */
	u64				acct_rss_mem1;
	/* Accumulated virtual memory usage: */
	u64				acct_vm_mem1;
	/* stime + utime since last update: */
	u64				acct_timexpd;
Linus Torvalds's avatar
Linus Torvalds committed
977
978
#endif
#ifdef CONFIG_CPUSETS
979
980
981
982
983
984
	/* Protected by ->alloc_lock: */
	nodemask_t			mems_allowed;
	/* Seqence number to catch updates: */
	seqcount_t			mems_allowed_seq;
	int				cpuset_mem_spread_rotor;
	int				cpuset_slab_spread_rotor;
Linus Torvalds's avatar
Linus Torvalds committed
985
#endif
986
#ifdef CONFIG_CGROUPS
987
988
989
990
	/* Control Group info protected by css_set_lock: */
	struct css_set __rcu		*cgroups;
	/* cg_list protected by css_set_lock and tsk->alloc_lock: */
	struct list_head		cg_list;
991
#endif
992
#ifdef CONFIG_X86_CPU_RESCTRL
993
	u32				closid;
994
	u32				rmid;
Fenghua Yu's avatar
Fenghua Yu committed
995
#endif
996
#ifdef CONFIG_FUTEX
997
	struct robust_list_head __user	*robust_list;
998
999
1000
#ifdef CONFIG_COMPAT
	struct compat_robust_list_head __user *compat_robust_list;
#endif