sched.h 43.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
#ifndef _LINUX_SCHED_H
#define _LINUX_SCHED_H

4
#include <uapi/linux/sched.h>
5

6
7
#include <linux/sched/prio.h>

Linus Torvalds's avatar
Linus Torvalds committed
8
#include <linux/capability.h>
9
#include <linux/mutex.h>
10
#include <linux/plist.h>
11
#include <linux/mm_types_task.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
13
14
#include <asm/ptrace.h>

#include <linux/sem.h>
15
#include <linux/shm.h>
Linus Torvalds's avatar
Linus Torvalds committed
16
#include <linux/signal.h>
17
#include <linux/signal_types.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
19
#include <linux/pid.h>
#include <linux/seccomp.h>
20
#include <linux/rculist.h>
Ingo Molnar's avatar
Ingo Molnar committed
21
#include <linux/rtmutex.h>
Linus Torvalds's avatar
Linus Torvalds committed
22

23
24
#include <linux/resource.h>
#include <linux/hrtimer.h>
Dmitry Vyukov's avatar
Dmitry Vyukov committed
25
#include <linux/kcov.h>
26
#include <linux/task_io_accounting.h>
Arjan van de Ven's avatar
Arjan van de Ven committed
27
#include <linux/latencytop.h>
28
#include <linux/cred.h>
29
#include <linux/gfp.h>
30
#include <linux/topology.h>
31
#include <linux/magic.h>
32
#include <linux/cgroup-defs.h>
33

34
35
#include <asm/current.h>

36
37
38
39
/* task_struct member predeclarations: */
struct audit_context;
struct autogroup;
struct backing_dev_info;
40
struct bio_list;
41
struct blk_plug;
42
struct cfs_rq;
43
struct filename;
44
45
46
47
struct fs_struct;
struct futex_pi_state;
struct io_context;
struct mempolicy;
48
struct nameidata;
49
50
51
52
53
54
55
56
57
struct nsproxy;
struct perf_event_context;
struct pid_namespace;
struct pipe_inode_info;
struct rcu_node;
struct reclaim_state;
struct robust_list_head;
struct sched_attr;
struct sched_param;
Ingo Molnar's avatar
Ingo Molnar committed
58
struct seq_file;
59
60
61
struct sighand_struct;
struct signal_struct;
struct task_delay_info;
62
struct task_group;
63
64
struct task_struct;
struct uts_namespace;
Linus Torvalds's avatar
Linus Torvalds committed
65

66
67
68
69
70
71
72
73
74
75
/*
 * Task state bitmask. NOTE! These bits are also
 * encoded in fs/proc/array.c: get_task_state().
 *
 * We have two separate sets of flags: task->state
 * is about runnability, while task->exit_state are
 * about the task exiting. Confusing, but this way
 * modifying one set can't modify the other one by
 * mistake.
 */
Linus Torvalds's avatar
Linus Torvalds committed
76
77
78
#define TASK_RUNNING		0
#define TASK_INTERRUPTIBLE	1
#define TASK_UNINTERRUPTIBLE	2
Matthew Wilcox's avatar
Matthew Wilcox committed
79
80
#define __TASK_STOPPED		4
#define __TASK_TRACED		8
81
/* in tsk->exit_state */
82
83
#define EXIT_DEAD		16
#define EXIT_ZOMBIE		32
84
#define EXIT_TRACE		(EXIT_ZOMBIE | EXIT_DEAD)
85
/* in tsk->state again */
86
#define TASK_DEAD		64
Matthew Wilcox's avatar
Matthew Wilcox committed
87
#define TASK_WAKEKILL		128
Peter Zijlstra's avatar
Peter Zijlstra committed
88
#define TASK_WAKING		256
89
#define TASK_PARKED		512
90
#define TASK_NOLOAD		1024
91
92
#define TASK_NEW		2048
#define TASK_STATE_MAX		4096
Matthew Wilcox's avatar
Matthew Wilcox committed
93

94
#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWPNn"
95

96
/* Convenience macros for the sake of set_current_state */
Matthew Wilcox's avatar
Matthew Wilcox committed
97
98
99
#define TASK_KILLABLE		(TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
#define TASK_STOPPED		(TASK_WAKEKILL | __TASK_STOPPED)
#define TASK_TRACED		(TASK_WAKEKILL | __TASK_TRACED)
Linus Torvalds's avatar
Linus Torvalds committed
100

101
102
#define TASK_IDLE		(TASK_UNINTERRUPTIBLE | TASK_NOLOAD)

103
104
/* Convenience macros for the sake of wake_up */
#define TASK_NORMAL		(TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
Matthew Wilcox's avatar
Matthew Wilcox committed
105
#define TASK_ALL		(TASK_NORMAL | __TASK_STOPPED | __TASK_TRACED)
106
107
108

/* get_task_state() */
#define TASK_REPORT		(TASK_RUNNING | TASK_INTERRUPTIBLE | \
Matthew Wilcox's avatar
Matthew Wilcox committed
109
				 TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
110
				 __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD)
111

Matthew Wilcox's avatar
Matthew Wilcox committed
112
113
#define task_is_traced(task)	((task->state & __TASK_TRACED) != 0)
#define task_is_stopped(task)	((task->state & __TASK_STOPPED) != 0)
114
#define task_is_stopped_or_traced(task)	\
Matthew Wilcox's avatar
Matthew Wilcox committed
115
			((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0)
116
#define task_contributes_to_load(task)	\
117
				((task->state & TASK_UNINTERRUPTIBLE) != 0 && \
118
119
				 (task->flags & PF_FROZEN) == 0 && \
				 (task->state & TASK_NOLOAD) == 0)
Linus Torvalds's avatar
Linus Torvalds committed
120

Peter Zijlstra's avatar
Peter Zijlstra committed
121
122
123
124
125
126
127
128
129
130
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP

#define __set_current_state(state_value)			\
	do {							\
		current->task_state_change = _THIS_IP_;		\
		current->state = (state_value);			\
	} while (0)
#define set_current_state(state_value)				\
	do {							\
		current->task_state_change = _THIS_IP_;		\
131
		smp_store_mb(current->state, (state_value));	\
Peter Zijlstra's avatar
Peter Zijlstra committed
132
133
134
	} while (0)

#else
135
136
137
138
139
/*
 * set_current_state() includes a barrier so that the write of current->state
 * is correctly serialised wrt the caller's subsequent test of whether to
 * actually sleep:
 *
140
 *   for (;;) {
141
 *	set_current_state(TASK_UNINTERRUPTIBLE);
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 *	if (!need_sleep)
 *		break;
 *
 *	schedule();
 *   }
 *   __set_current_state(TASK_RUNNING);
 *
 * If the caller does not need such serialisation (because, for instance, the
 * condition test and condition change and wakeup are under the same lock) then
 * use __set_current_state().
 *
 * The above is typically ordered against the wakeup, which does:
 *
 *	need_sleep = false;
 *	wake_up_state(p, TASK_UNINTERRUPTIBLE);
 *
 * Where wake_up_state() (and all other wakeup primitives) imply enough
 * barriers to order the store of the variable against wakeup.
 *
 * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
 * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
 * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
164
 *
165
 * This is obviously fine, since they both store the exact same value.
166
 *
167
 * Also see the comments of try_to_wake_up().
168
 */
Peter Zijlstra's avatar
Peter Zijlstra committed
169
#define __set_current_state(state_value)		\
Linus Torvalds's avatar
Linus Torvalds committed
170
	do { current->state = (state_value); } while (0)
Peter Zijlstra's avatar
Peter Zijlstra committed
171
#define set_current_state(state_value)			\
172
	smp_store_mb(current->state, (state_value))
Linus Torvalds's avatar
Linus Torvalds committed
173

Peter Zijlstra's avatar
Peter Zijlstra committed
174
175
#endif

Linus Torvalds's avatar
Linus Torvalds committed
176
177
178
/* Task command name length */
#define TASK_COMM_LEN 16

179
180
extern cpumask_var_t cpu_isolated_map;

181
extern int runqueue_is_locked(int cpu);
Ingo Molnar's avatar
Ingo Molnar committed
182

Linus Torvalds's avatar
Linus Torvalds committed
183
184
185
186
extern void update_process_times(int user);
extern void scheduler_tick(void);

#define	MAX_SCHEDULE_TIMEOUT	LONG_MAX
187
extern signed long schedule_timeout(signed long timeout);
188
extern signed long schedule_timeout_interruptible(signed long timeout);
Matthew Wilcox's avatar
Matthew Wilcox committed
189
extern signed long schedule_timeout_killable(signed long timeout);
190
extern signed long schedule_timeout_uninterruptible(signed long timeout);
191
extern signed long schedule_timeout_idle(signed long timeout);
Linus Torvalds's avatar
Linus Torvalds committed
192
asmlinkage void schedule(void);
193
extern void schedule_preempt_disabled(void);
Linus Torvalds's avatar
Linus Torvalds committed
194

195
196
extern int __must_check io_schedule_prepare(void);
extern void io_schedule_finish(int token);
197
extern long io_schedule_timeout(long timeout);
198
extern void io_schedule(void);
199

200
/**
201
 * struct prev_cputime - snaphsot of system and user cputime
202
203
 * @utime: time spent in user mode
 * @stime: time spent in system mode
204
 * @lock: protects the above two fields
205
 *
206
207
 * Stores previous user/system time values such that we can guarantee
 * monotonicity.
208
 */
209
210
struct prev_cputime {
#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
211
212
	u64 utime;
	u64 stime;
213
214
	raw_spinlock_t lock;
#endif
215
216
};

217
218
/**
 * struct task_cputime - collected CPU time counts
219
220
 * @utime:		time spent in user mode, in nanoseconds
 * @stime:		time spent in kernel mode, in nanoseconds
221
 * @sum_exec_runtime:	total time spent on the CPU, in nanoseconds
222
 *
223
224
225
 * This structure groups together three kinds of CPU time that are tracked for
 * threads and thread groups.  Most things considering CPU time want to group
 * these counts together and treat all three of them in parallel.
226
227
 */
struct task_cputime {
228
229
	u64 utime;
	u64 stime;
230
231
	unsigned long long sum_exec_runtime;
};
232

233
234
/* Alternate field names when used to cache expirations. */
#define virt_exp	utime
235
#define prof_exp	stime
236
237
#define sched_exp	sum_exec_runtime

238
#include <linux/rwsem.h>
Linus Torvalds's avatar
Linus Torvalds committed
239

240
#ifdef CONFIG_SCHED_INFO
Linus Torvalds's avatar
Linus Torvalds committed
241
242
struct sched_info {
	/* cumulative counters */
243
	unsigned long pcount;	      /* # of times run on this cpu */
244
	unsigned long long run_delay; /* time spent waiting on a runqueue */
Linus Torvalds's avatar
Linus Torvalds committed
245
246

	/* timestamps */
247
248
	unsigned long long last_arrival,/* when we last ran on a cpu */
			   last_queued;	/* when we were last queued to run */
Linus Torvalds's avatar
Linus Torvalds committed
249
};
250
#endif /* CONFIG_SCHED_INFO */
Linus Torvalds's avatar
Linus Torvalds committed
251

252
253
254
255
256
257
258
259
260
261
/*
 * Integer metrics need fixed point arithmetic, e.g., sched/fair
 * has a few: load, load_avg, util_avg, freq, and capacity.
 *
 * We define a basic fixed point arithmetic range, and then formalize
 * all these metrics based on that basic range.
 */
# define SCHED_FIXEDPOINT_SHIFT	10
# define SCHED_FIXEDPOINT_SCALE	(1L << SCHED_FIXEDPOINT_SHIFT)

262
#ifdef ARCH_HAS_PREFETCH_SWITCH_STACK
263
extern void prefetch_stack(struct task_struct *t);
264
265
266
#else
static inline void prefetch_stack(struct task_struct *t) { }
#endif
Linus Torvalds's avatar
Linus Torvalds committed
267

Ingo Molnar's avatar
Ingo Molnar committed
268
struct load_weight {
269
270
	unsigned long weight;
	u32 inv_weight;
Ingo Molnar's avatar
Ingo Molnar committed
271
272
};

273
/*
274
275
276
277
278
279
280
281
282
 * The load_avg/util_avg accumulates an infinite geometric series
 * (see __update_load_avg() in kernel/sched/fair.c).
 *
 * [load_avg definition]
 *
 *   load_avg = runnable% * scale_load_down(load)
 *
 * where runnable% is the time ratio that a sched_entity is runnable.
 * For cfs_rq, it is the aggregated load_avg of all runnable and
283
 * blocked sched_entities.
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
 *
 * load_avg may also take frequency scaling into account:
 *
 *   load_avg = runnable% * scale_load_down(load) * freq%
 *
 * where freq% is the CPU frequency normalized to the highest frequency.
 *
 * [util_avg definition]
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE
 *
 * where running% is the time ratio that a sched_entity is running on
 * a CPU. For cfs_rq, it is the aggregated util_avg of all runnable
 * and blocked sched_entities.
 *
 * util_avg may also factor frequency scaling and CPU capacity scaling:
 *
 *   util_avg = running% * SCHED_CAPACITY_SCALE * freq% * capacity%
 *
 * where freq% is the same as above, and capacity% is the CPU capacity
 * normalized to the greatest capacity (due to uarch differences, etc).
 *
 * N.B., the above ratios (runnable%, running%, freq%, and capacity%)
 * themselves are in the range of [0, 1]. To do fixed point arithmetics,
 * we therefore scale them to as large a range as necessary. This is for
 * example reflected by util_avg's SCHED_CAPACITY_SCALE.
 *
 * [Overflow issue]
 *
 * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
 * with the highest load (=88761), always runnable on a single cfs_rq,
 * and should not overflow as the number already hits PID_MAX_LIMIT.
 *
 * For all other cases (including 32-bit kernels), struct load_weight's
 * weight will overflow first before we do, because:
 *
 *    Max(load_avg) <= Max(load.weight)
 *
 * Then it is the load_weight's responsibility to consider overflow
 * issues.
324
 */
325
struct sched_avg {
326
327
328
	u64 last_update_time, load_sum;
	u32 util_sum, period_contrib;
	unsigned long load_avg, util_avg;
329
330
};

331
#ifdef CONFIG_SCHEDSTATS
332
struct sched_statistics {
Ingo Molnar's avatar
Ingo Molnar committed
333
	u64			wait_start;
334
	u64			wait_max;
335
336
	u64			wait_count;
	u64			wait_sum;
337
338
	u64			iowait_count;
	u64			iowait_sum;
339

Ingo Molnar's avatar
Ingo Molnar committed
340
341
	u64			sleep_start;
	u64			sleep_max;
342
343
344
	s64			sum_sleep_runtime;

	u64			block_start;
Ingo Molnar's avatar
Ingo Molnar committed
345
346
	u64			block_max;
	u64			exec_max;
347
	u64			slice_max;
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363

	u64			nr_migrations_cold;
	u64			nr_failed_migrations_affine;
	u64			nr_failed_migrations_running;
	u64			nr_failed_migrations_hot;
	u64			nr_forced_migrations;

	u64			nr_wakeups;
	u64			nr_wakeups_sync;
	u64			nr_wakeups_migrate;
	u64			nr_wakeups_local;
	u64			nr_wakeups_remote;
	u64			nr_wakeups_affine;
	u64			nr_wakeups_affine_attempts;
	u64			nr_wakeups_passive;
	u64			nr_wakeups_idle;
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
};
#endif

struct sched_entity {
	struct load_weight	load;		/* for load-balancing */
	struct rb_node		run_node;
	struct list_head	group_node;
	unsigned int		on_rq;

	u64			exec_start;
	u64			sum_exec_runtime;
	u64			vruntime;
	u64			prev_sum_exec_runtime;

	u64			nr_migrations;

#ifdef CONFIG_SCHEDSTATS
	struct sched_statistics statistics;
382
383
#endif

Ingo Molnar's avatar
Ingo Molnar committed
384
#ifdef CONFIG_FAIR_GROUP_SCHED
385
	int			depth;
Ingo Molnar's avatar
Ingo Molnar committed
386
387
388
389
390
391
	struct sched_entity	*parent;
	/* rq on which this entity is (to be) queued: */
	struct cfs_rq		*cfs_rq;
	/* rq "owned" by this entity/group: */
	struct cfs_rq		*my_q;
#endif
392

393
#ifdef CONFIG_SMP
394
395
396
397
398
399
400
	/*
	 * Per entity load average tracking.
	 *
	 * Put into separate cache line so it does not
	 * collide with read-mostly values above.
	 */
	struct sched_avg	avg ____cacheline_aligned_in_smp;
401
#endif
Ingo Molnar's avatar
Ingo Molnar committed
402
};
403

Peter Zijlstra's avatar
Peter Zijlstra committed
404
405
struct sched_rt_entity {
	struct list_head run_list;
406
	unsigned long timeout;
407
	unsigned long watchdog_stamp;
408
	unsigned int time_slice;
409
410
	unsigned short on_rq;
	unsigned short on_list;
Peter Zijlstra's avatar
Peter Zijlstra committed
411

412
	struct sched_rt_entity *back;
413
#ifdef CONFIG_RT_GROUP_SCHED
Peter Zijlstra's avatar
Peter Zijlstra committed
414
415
416
417
418
419
	struct sched_rt_entity	*parent;
	/* rq on which this entity is (to be) queued: */
	struct rt_rq		*rt_rq;
	/* rq "owned" by this entity/group: */
	struct rt_rq		*my_q;
#endif
Peter Zijlstra's avatar
Peter Zijlstra committed
420
421
};

422
423
424
425
426
struct sched_dl_entity {
	struct rb_node	rb_node;

	/*
	 * Original scheduling parameters. Copied here from sched_attr
427
428
	 * during sched_setattr(), they will remain the same until
	 * the next sched_setattr().
429
430
431
	 */
	u64 dl_runtime;		/* maximum runtime for each instance	*/
	u64 dl_deadline;	/* relative deadline of each instance	*/
432
	u64 dl_period;		/* separation of two instances (period) */
433
	u64 dl_bw;		/* dl_runtime / dl_deadline		*/
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450

	/*
	 * Actual scheduling parameters. Initialized with the values above,
	 * they are continously updated during task execution. Note that
	 * the remaining runtime could be < 0 in case we are in overrun.
	 */
	s64 runtime;		/* remaining runtime for this instance	*/
	u64 deadline;		/* absolute deadline for this instance	*/
	unsigned int flags;	/* specifying the scheduler behaviour	*/

	/*
	 * Some bool flags:
	 *
	 * @dl_throttled tells if we exhausted the runtime. If so, the
	 * task has to wait for a replenishment to be performed at the
	 * next firing of dl_timer.
	 *
451
452
	 * @dl_boosted tells if we are boosted due to DI. If so we are
	 * outside bandwidth enforcement mechanism (but only until we
453
454
455
456
	 * exit the critical section);
	 *
	 * @dl_yielded tells if task gave up the cpu before consuming
	 * all its available runtime during the last job.
457
	 */
458
	int dl_throttled, dl_boosted, dl_yielded;
459
460
461
462
463
464
465

	/*
	 * Bandwidth enforcement timer. Each -deadline task has its
	 * own bandwidth to be enforced, thus we need one timer per task.
	 */
	struct hrtimer dl_timer;
};
466

467
468
union rcu_special {
	struct {
469
470
471
472
473
474
		u8 blocked;
		u8 need_qs;
		u8 exp_need_qs;
		u8 pad;	/* Otherwise the compiler can store garbage here. */
	} b; /* Bits. */
	u32 s; /* Set of bits. */
475
};
476

Peter Zijlstra's avatar
Peter Zijlstra committed
477
478
479
enum perf_event_task_context {
	perf_invalid_context = -1,
	perf_hw_context = 0,
480
	perf_sw_context,
Peter Zijlstra's avatar
Peter Zijlstra committed
481
482
483
	perf_nr_task_contexts,
};

484
485
486
487
struct wake_q_node {
	struct wake_q_node *next;
};

488
489
490
491
492
493
494
495
496
497
/* Track pages that require TLB flushes */
struct tlbflush_unmap_batch {
	/*
	 * Each bit set is a CPU that potentially has a TLB entry for one of
	 * the PFNs being flushed. See set_tlb_ubc_flush_pending().
	 */
	struct cpumask cpumask;

	/* True if any bit in cpumask is set */
	bool flush_required;
498
499
500
501
502
503
504

	/*
	 * If true then the PTE was dirty when unmapped. The entry must be
	 * flushed before IO is initiated or a stale TLB entry potentially
	 * allows an update without redirtying the page.
	 */
	bool writable;
505
506
};

Linus Torvalds's avatar
Linus Torvalds committed
507
struct task_struct {
508
509
510
511
512
513
514
#ifdef CONFIG_THREAD_INFO_IN_TASK
	/*
	 * For reasons of header soup (see current_thread_info()), this
	 * must be the first element of task_struct.
	 */
	struct thread_info thread_info;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
515
	volatile long state;	/* -1 unrunnable, 0 runnable, >0 stopped */
Roman Zippel's avatar
Roman Zippel committed
516
	void *stack;
Linus Torvalds's avatar
Linus Torvalds committed
517
	atomic_t usage;
518
519
	unsigned int flags;	/* per process flags, defined below */
	unsigned int ptrace;
Linus Torvalds's avatar
Linus Torvalds committed
520

521
#ifdef CONFIG_SMP
522
	struct llist_node wake_entry;
523
	int on_cpu;
524
525
526
#ifdef CONFIG_THREAD_INFO_IN_TASK
	unsigned int cpu;	/* current CPU */
#endif
527
	unsigned int wakee_flips;
528
	unsigned long wakee_flip_decay_ts;
529
	struct task_struct *last_wakee;
530
531

	int wake_cpu;
532
#endif
Peter Zijlstra's avatar
Peter Zijlstra committed
533
	int on_rq;
534

535
	int prio, static_prio, normal_prio;
536
	unsigned int rt_priority;
537
	const struct sched_class *sched_class;
Ingo Molnar's avatar
Ingo Molnar committed
538
	struct sched_entity se;
Peter Zijlstra's avatar
Peter Zijlstra committed
539
	struct sched_rt_entity rt;
540
541
542
#ifdef CONFIG_CGROUP_SCHED
	struct task_group *sched_task_group;
#endif
543
	struct sched_dl_entity dl;
Linus Torvalds's avatar
Linus Torvalds committed
544

545
546
547
548
549
#ifdef CONFIG_PREEMPT_NOTIFIERS
	/* list of struct preempt_notifier: */
	struct hlist_head preempt_notifiers;
#endif

550
#ifdef CONFIG_BLK_DEV_IO_TRACE
551
	unsigned int btrace_seq;
552
#endif
Linus Torvalds's avatar
Linus Torvalds committed
553

554
	unsigned int policy;
555
	int nr_cpus_allowed;
Linus Torvalds's avatar
Linus Torvalds committed
556
557
	cpumask_t cpus_allowed;

Paul E. McKenney's avatar
Paul E. McKenney committed
558
#ifdef CONFIG_PREEMPT_RCU
Paul E. McKenney's avatar
Paul E. McKenney committed
559
	int rcu_read_lock_nesting;
560
	union rcu_special rcu_read_unlock_special;
561
	struct list_head rcu_node_entry;
Paul E. McKenney's avatar
Paul E. McKenney committed
562
	struct rcu_node *rcu_blocked_node;
563
#endif /* #ifdef CONFIG_PREEMPT_RCU */
Paul E. McKenney's avatar
Paul E. McKenney committed
564
565
566
567
#ifdef CONFIG_TASKS_RCU
	unsigned long rcu_tasks_nvcsw;
	bool rcu_tasks_holdout;
	struct list_head rcu_tasks_holdout_list;
568
	int rcu_tasks_idle_cpu;
Paul E. McKenney's avatar
Paul E. McKenney committed
569
#endif /* #ifdef CONFIG_TASKS_RCU */
Paul E. McKenney's avatar
Paul E. McKenney committed
570

571
#ifdef CONFIG_SCHED_INFO
Linus Torvalds's avatar
Linus Torvalds committed
572
573
574
575
	struct sched_info sched_info;
#endif

	struct list_head tasks;
576
#ifdef CONFIG_SMP
577
	struct plist_node pushable_tasks;
578
	struct rb_node pushable_dl_tasks;
579
#endif
Linus Torvalds's avatar
Linus Torvalds committed
580
581

	struct mm_struct *mm, *active_mm;
582
583
584
585

	/* Per-thread vma caching: */
	struct vmacache vmacache;

586
587
588
#if defined(SPLIT_RSS_COUNTING)
	struct task_rss_stat	rss_stat;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
589
/* task state */
590
	int exit_state;
Linus Torvalds's avatar
Linus Torvalds committed
591
592
	int exit_code, exit_signal;
	int pdeath_signal;  /*  The signal sent when the parent dies  */
593
	unsigned long jobctl;	/* JOBCTL_*, siglock protected */
594
595

	/* Used for emulating ABI behavior of previous Linux versions */
596
	unsigned int personality;
597

598
	/* scheduler bits, serialized by scheduler locks */
599
	unsigned sched_reset_on_fork:1;
600
	unsigned sched_contributes_to_load:1;
601
	unsigned sched_migrated:1;
602
	unsigned sched_remote_wakeup:1;
603
604
605
606
607
	unsigned :0; /* force alignment to the next boundary */

	/* unserialized, strictly 'current' */
	unsigned in_execve:1; /* bit to tell LSMs we're in execve */
	unsigned in_iowait:1;
608
609
610
#if !defined(TIF_RESTORE_SIGMASK)
	unsigned restore_sigmask:1;
#endif
611
612
#ifdef CONFIG_MEMCG
	unsigned memcg_may_oom:1;
613
#ifndef CONFIG_SLOB
614
615
	unsigned memcg_kmem_skip_account:1;
#endif
616
#endif
617
618
619
#ifdef CONFIG_COMPAT_BRK
	unsigned brk_randomized:1;
#endif
620

621
622
	unsigned long atomic_flags; /* Flags needing atomic access. */

623
624
	struct restart_block restart_block;

Linus Torvalds's avatar
Linus Torvalds committed
625
626
	pid_t pid;
	pid_t tgid;
627

628
#ifdef CONFIG_CC_STACKPROTECTOR
629
630
	/* Canary value for the -fstack-protector gcc feature */
	unsigned long stack_canary;
631
#endif
632
	/*
Linus Torvalds's avatar
Linus Torvalds committed
633
	 * pointers to (original) parent process, youngest child, younger sibling,
634
	 * older sibling, respectively.  (p->father can be replaced with
Roland McGrath's avatar
Roland McGrath committed
635
	 * p->real_parent->pid)
Linus Torvalds's avatar
Linus Torvalds committed
636
	 */
637
638
	struct task_struct __rcu *real_parent; /* real parent process */
	struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */
Linus Torvalds's avatar
Linus Torvalds committed
639
	/*
Roland McGrath's avatar
Roland McGrath committed
640
	 * children/sibling forms the list of my natural children
Linus Torvalds's avatar
Linus Torvalds committed
641
642
643
644
645
	 */
	struct list_head children;	/* list of my children */
	struct list_head sibling;	/* linkage in my parent's children list */
	struct task_struct *group_leader;	/* threadgroup leader */

Roland McGrath's avatar
Roland McGrath committed
646
647
648
649
650
651
652
653
	/*
	 * ptraced is the list of tasks this task is using ptrace on.
	 * This includes both natural children and PTRACE_ATTACH targets.
	 * p->ptrace_entry is p's link on the p->parent->ptraced list.
	 */
	struct list_head ptraced;
	struct list_head ptrace_entry;

Linus Torvalds's avatar
Linus Torvalds committed
654
	/* PID/PID hash table linkage. */
655
	struct pid_link pids[PIDTYPE_MAX];
656
	struct list_head thread_group;
657
	struct list_head thread_node;
Linus Torvalds's avatar
Linus Torvalds committed
658
659
660
661
662

	struct completion *vfork_done;		/* for vfork() */
	int __user *set_child_tid;		/* CLONE_CHILD_SETTID */
	int __user *clear_child_tid;		/* CLONE_CHILD_CLEARTID */

663
	u64 utime, stime;
664
#ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
665
	u64 utimescaled, stimescaled;
666
#endif
667
	u64 gtime;
668
	struct prev_cputime prev_cputime;
669
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
670
	seqcount_t vtime_seqcount;
671
672
	unsigned long long vtime_snap;
	enum {
673
674
675
		/* Task is sleeping or running in a CPU with VTIME inactive */
		VTIME_INACTIVE = 0,
		/* Task runs in userspace in a CPU with VTIME active */
676
		VTIME_USER,
677
		/* Task runs in kernelspace in a CPU with VTIME active */
678
679
		VTIME_SYS,
	} vtime_snap_whence;
680
#endif
681
682

#ifdef CONFIG_NO_HZ_FULL
683
	atomic_t tick_dep_mask;
684
#endif
Linus Torvalds's avatar
Linus Torvalds committed
685
	unsigned long nvcsw, nivcsw; /* context switch counts */
686
	u64 start_time;		/* monotonic time in nsec */
687
	u64 real_start_time;	/* boot based time in nsec */
Linus Torvalds's avatar
Linus Torvalds committed
688
689
690
/* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
	unsigned long min_flt, maj_flt;

691
#ifdef CONFIG_POSIX_TIMERS
692
	struct task_cputime cputime_expires;
Linus Torvalds's avatar
Linus Torvalds committed
693
	struct list_head cpu_timers[3];
694
#endif
Linus Torvalds's avatar
Linus Torvalds committed
695
696

/* process credentials */
697
	const struct cred __rcu *ptracer_cred; /* Tracer's credentials at attach */
Arnd Bergmann's avatar
Arnd Bergmann committed
698
	const struct cred __rcu *real_cred; /* objective and real subjective task
699
					 * credentials (COW) */
Arnd Bergmann's avatar
Arnd Bergmann committed
700
	const struct cred __rcu *cred;	/* effective (overridable) subjective task
701
					 * credentials (COW) */
702
703
704
	char comm[TASK_COMM_LEN]; /* executable name excluding path
				     - access with [gs]et_task_comm (which lock
				       it with task_lock())
705
				     - initialized normally by setup_new_exec */
Linus Torvalds's avatar
Linus Torvalds committed
706
/* file system info */
707
	struct nameidata *nameidata;
708
#ifdef CONFIG_SYSVIPC
Linus Torvalds's avatar
Linus Torvalds committed
709
710
/* ipc stuff */
	struct sysv_sem sysvsem;
711
	struct sysv_shm sysvshm;
712
#endif
713
#ifdef CONFIG_DETECT_HUNG_TASK
714
715
716
/* hung task detection */
	unsigned long last_switch_count;
#endif
Linus Torvalds's avatar
Linus Torvalds committed
717
718
719
720
/* filesystem information */
	struct fs_struct *fs;
/* open file information */
	struct files_struct *files;
721
/* namespaces */
722
	struct nsproxy *nsproxy;
Linus Torvalds's avatar
Linus Torvalds committed
723
724
725
726
727
/* signal handlers */
	struct signal_struct *signal;
	struct sighand_struct *sighand;

	sigset_t blocked, real_blocked;
728
	sigset_t saved_sigmask;	/* restored if set_restore_sigmask() was used */
Linus Torvalds's avatar
Linus Torvalds committed
729
730
731
732
	struct sigpending pending;

	unsigned long sas_ss_sp;
	size_t sas_ss_size;
733
	unsigned sas_ss_flags;
734

735
	struct callback_head *task_works;
736

Linus Torvalds's avatar
Linus Torvalds committed
737
	struct audit_context *audit_context;
Al Viro's avatar
Al Viro committed
738
#ifdef CONFIG_AUDITSYSCALL
739
	kuid_t loginuid;
740
	unsigned int sessionid;
Al Viro's avatar
Al Viro committed
741
#endif
742
	struct seccomp seccomp;
Linus Torvalds's avatar
Linus Torvalds committed
743
744
745
746

/* Thread group tracking */
   	u32 parent_exec_id;
   	u32 self_exec_id;
747
748
/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
 * mempolicy */
Linus Torvalds's avatar
Linus Torvalds committed
749
750
	spinlock_t alloc_lock;

751
	/* Protection of the PI data structures: */
752
	raw_spinlock_t pi_lock;
753

754
755
	struct wake_q_node wake_q;

Ingo Molnar's avatar
Ingo Molnar committed
756
757
#ifdef CONFIG_RT_MUTEXES
	/* PI waiters blocked on a rt_mutex held by this task */
758
759
	struct rb_root pi_waiters;
	struct rb_node *pi_waiters_leftmost;
Ingo Molnar's avatar
Ingo Molnar committed
760
761
762
763
	/* Deadlock detection and priority inheritance handling */
	struct rt_mutex_waiter *pi_blocked_on;
#endif

764
765
766
767
#ifdef CONFIG_DEBUG_MUTEXES
	/* mutex deadlock detection */
	struct mutex_waiter *blocked_on;
#endif
768
769
770
771
#ifdef CONFIG_TRACE_IRQFLAGS
	unsigned int irq_events;
	unsigned long hardirq_enable_ip;
	unsigned long hardirq_disable_ip;
772
	unsigned int hardirq_enable_event;
773
	unsigned int hardirq_disable_event;
774
775
	int hardirqs_enabled;
	int hardirq_context;
776
777
	unsigned long softirq_disable_ip;
	unsigned long softirq_enable_ip;
778
	unsigned int softirq_disable_event;
779
	unsigned int softirq_enable_event;
780
	int softirqs_enabled;
781
782
	int softirq_context;
#endif
Ingo Molnar's avatar
Ingo Molnar committed
783
#ifdef CONFIG_LOCKDEP
784
# define MAX_LOCK_DEPTH 48UL
Ingo Molnar's avatar
Ingo Molnar committed
785
786
787
	u64 curr_chain_key;
	int lockdep_depth;
	unsigned int lockdep_recursion;
788
	struct held_lock held_locks[MAX_LOCK_DEPTH];
789
	gfp_t lockdep_reclaim_gfp;
Ingo Molnar's avatar
Ingo Molnar committed
790
#endif
791
792
793
#ifdef CONFIG_UBSAN
	unsigned int in_ubsan;
#endif
794

Linus Torvalds's avatar
Linus Torvalds committed
795
796
797
/* journalling filesystem info */
	void *journal_info;

798
/* stacked block device info */
799
	struct bio_list *bio_list;
800

801
802
803
804
805
#ifdef CONFIG_BLOCK
/* stack plugging */
	struct blk_plug *plug;
#endif

Linus Torvalds's avatar
Linus Torvalds committed
806
807
808
809
810
811
812
813
814
/* VM state */
	struct reclaim_state *reclaim_state;

	struct backing_dev_info *backing_dev_info;

	struct io_context *io_context;

	unsigned long ptrace_message;
	siginfo_t *last_siginfo; /* For ptrace use.  */
815
	struct task_io_accounting ioac;
816
#if defined(CONFIG_TASK_XACCT)
Linus Torvalds's avatar
Linus Torvalds committed
817
818
	u64 acct_rss_mem1;	/* accumulated rss usage */
	u64 acct_vm_mem1;	/* accumulated virtual memory usage */
819
	u64 acct_timexpd;	/* stime + utime since last update */
Linus Torvalds's avatar
Linus Torvalds committed
820
821
#endif
#ifdef CONFIG_CPUSETS
822
	nodemask_t mems_allowed;	/* Protected by alloc_lock */
823
	seqcount_t mems_allowed_seq;	/* Seqence no to catch updates */
824
	int cpuset_mem_spread_rotor;
825
	int cpuset_slab_spread_rotor;
Linus Torvalds's avatar
Linus Torvalds committed
826
#endif
827
#ifdef CONFIG_CGROUPS
828
	/* Control Group info protected by css_set_lock */
Arnd Bergmann's avatar
Arnd Bergmann committed
829
	struct css_set __rcu *cgroups;
830
831
	/* cg_list protected by css_set_lock and tsk->alloc_lock */
	struct list_head cg_list;
832
#endif
Fenghua Yu's avatar
Fenghua Yu committed
833
834
835
#ifdef CONFIG_INTEL_RDT_A
	int closid;
#endif
836
#ifdef CONFIG_FUTEX
837
	struct robust_list_head __user *robust_list;
838
839
840
#ifdef CONFIG_COMPAT
	struct compat_robust_list_head __user *compat_robust_list;
#endif
841
842
	struct list_head pi_state_list;
	struct futex_pi_state *pi_state_cache;
843
#endif
844
#ifdef CONFIG_PERF_EVENTS
Peter Zijlstra's avatar
Peter Zijlstra committed
845
	struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
846
847
	struct mutex perf_event_mutex;
	struct list_head perf_event_list;
848
#endif
849
850
851
#ifdef CONFIG_DEBUG_PREEMPT
	unsigned long preempt_disable_ip;
#endif
852
#ifdef CONFIG_NUMA
853
	struct mempolicy *mempolicy;	/* Protected by alloc_lock */
854
	short il_next;
855
	short pref_node_fork;
856
#endif
857
858
859
#ifdef CONFIG_NUMA_BALANCING
	int numa_scan_seq;
	unsigned int numa_scan_period;
860
	unsigned int numa_scan_period_max;
861
	int numa_preferred_nid;
862
	unsigned long numa_migrate_retry;
863
	u64 node_stamp;			/* migration stamp  */
864
865
	u64 last_task_numa_placement;
	u64 last_sum_exec_runtime;
866
	struct callback_head numa_work;
867

868
869
870
	struct list_head numa_entry;
	struct numa_group *numa_group;

871
	/*
872
873
874
875
876
877
878
879
880
881
882
883
	 * numa_faults is an array split into four regions:
	 * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
	 * in this precise order.
	 *
	 * faults_memory: Exponential decaying average of faults on a per-node
	 * basis. Scheduling placement decisions are made based on these
	 * counts. The values remain static for the duration of a PTE scan.
	 * faults_cpu: Track the nodes the process was running on when a NUMA
	 * hinting fault was incurred.
	 * faults_memory_buffer and faults_cpu_buffer: Record faults per node
	 * during the current scan window. When the scan completes, the counts
	 * in faults_memory and faults_cpu decay and these values are copied.
884
	 */
885
	unsigned long *numa_faults;
886
	unsigned long total_numa_faults;
887

888
889
	/*
	 * numa_faults_locality tracks if faults recorded during the last
890
891
892
	 * scan window were remote/local or failed to migrate. The task scan
	 * period is adapted based on the locality of the faults with different
	 * weights depending on whether they were shared or private faults
893
	 */
894
	unsigned long numa_faults_locality[3];
895

Ingo Molnar's avatar
Ingo Molnar committed
896
	unsigned long numa_pages_migrated;
897
898
#endif /* CONFIG_NUMA_BALANCING */

899
900
901
902
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
	struct tlbflush_unmap_batch tlb_ubc;
#endif

Ingo Molnar's avatar
Ingo Molnar committed
903
	struct rcu_head rcu;
904
905
906
907
908

	/*
	 * cache last used pipe for splice
	 */
	struct pipe_inode_info *splice_pipe;
909
910
911

	struct page_frag task_frag;

912
913
#ifdef CONFIG_TASK_DELAY_ACCT
	struct task_delay_info		*delays;
914
#endif
915

916
917
#ifdef CONFIG_FAULT_INJECTION
	int make_it_fail;
918
#endif
919
920
921
922
923
924
	/*
	 * when (nr_dirtied >= nr_dirtied_pause), it's time to call
	 * balance_dirty_pages() for some dirty throttling pause
	 */
	int nr_dirtied;
	int nr_dirtied_pause;
925
	unsigned long dirty_paused_when; /* start of a write-and-pause period */
926

Arjan van de Ven's avatar
Arjan van de Ven committed
927
928
929
930
#ifdef CONFIG_LATENCYTOP
	int latency_record_count;
	struct latency_record latency_record[LT_SAVECOUNT];
#endif
931
932
933
934
	/*
	 * time slack values; these are used to round up poll() and
	 * select() etc timeout values. These are in nanoseconds.
	 */
935
936
	u64 timer_slack_ns;
	u64 default_timer_slack_ns;
937

938
939
940
#ifdef CONFIG_KASAN
	unsigned int kasan_depth;
#endif
941
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
942
	/* Index of current stored address in ret_stack */
943
944
945
	int curr_ret_stack;
	/* Stack of return addresses for return function tracing */
	struct ftrace_ret_stack	*ret_stack;
946
947
	/* time stamp for last schedule */
	unsigned long long ftrace_timestamp;
948
949
950
951
952
	/*
	 * Number of functions that haven't been traced
	 * because of depth overrun.
	 */
	atomic_t trace_overrun;
953
954
	/* Pause for the tracing */
	atomic_t tracing_graph_pause;
955
#endif
956
957
958
#ifdef CONFIG_TRACING
	/* state flags for use by tracers */
	unsigned long trace;
959
	/* bitmask and counter of trace recursion */
<