From owner-svn-src-user@FreeBSD.ORG Wed May 20 07:31:12 2009 Return-Path: Delivered-To: svn-src-user@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id 9E646106564A; Wed, 20 May 2009 07:31:12 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 8BA0A8FC16; Wed, 20 May 2009 07:31:12 +0000 (UTC) (envelope-from kmacy@FreeBSD.org) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.3/8.14.3) with ESMTP id n4K7VC6Y023119; Wed, 20 May 2009 07:31:12 GMT (envelope-from kmacy@svn.freebsd.org) Received: (from kmacy@localhost) by svn.freebsd.org (8.14.3/8.14.3/Submit) id n4K7VCYZ023111; Wed, 20 May 2009 07:31:12 GMT (envelope-from kmacy@svn.freebsd.org) Message-Id: <200905200731.n4K7VCYZ023111@svn.freebsd.org> From: Kip Macy Date: Wed, 20 May 2009 07:31:12 +0000 (UTC) To: src-committers@freebsd.org, svn-src-user@freebsd.org X-SVN-Group: user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r192431 - in user/kmacy/releng_7_2_fcs/sys: ddb kern sys X-BeenThere: svn-src-user@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the experimental " user" src tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 20 May 2009 07:31:12 -0000 Author: kmacy Date: Wed May 20 07:31:11 2009 New Revision: 192431 URL: http://svn.freebsd.org/changeset/base/192431 Log: - Update locks to newer lock profiling and witness - Update ddb for support Modified: user/kmacy/releng_7_2_fcs/sys/ddb/db_command.c user/kmacy/releng_7_2_fcs/sys/ddb/db_command.h user/kmacy/releng_7_2_fcs/sys/ddb/db_main.c user/kmacy/releng_7_2_fcs/sys/ddb/ddb.h user/kmacy/releng_7_2_fcs/sys/kern/kern_mutex.c user/kmacy/releng_7_2_fcs/sys/kern/kern_rmlock.c user/kmacy/releng_7_2_fcs/sys/kern/kern_rwlock.c user/kmacy/releng_7_2_fcs/sys/kern/kern_sx.c user/kmacy/releng_7_2_fcs/sys/kern/subr_lock.c user/kmacy/releng_7_2_fcs/sys/kern/subr_sleepqueue.c user/kmacy/releng_7_2_fcs/sys/kern/subr_turnstile.c user/kmacy/releng_7_2_fcs/sys/kern/subr_witness.c user/kmacy/releng_7_2_fcs/sys/sys/_lock.h user/kmacy/releng_7_2_fcs/sys/sys/_rwlock.h user/kmacy/releng_7_2_fcs/sys/sys/lock.h user/kmacy/releng_7_2_fcs/sys/sys/lock_profile.h user/kmacy/releng_7_2_fcs/sys/sys/proc.h user/kmacy/releng_7_2_fcs/sys/sys/rwlock.h Modified: user/kmacy/releng_7_2_fcs/sys/ddb/db_command.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/ddb/db_command.c Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/ddb/db_command.c Wed May 20 07:31:11 2009 (r192431) @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include #include @@ -63,10 +64,6 @@ db_addr_t db_last_addr; db_addr_t db_prev; db_addr_t db_next; -SET_DECLARE(db_cmd_set, struct command); -SET_DECLARE(db_show_cmd_set, struct command); -SET_DECLARE(db_show_all_cmd_set, struct command); - static db_cmdfcn_t db_fncall; static db_cmdfcn_t db_gdb; static db_cmdfcn_t db_halt; @@ -81,30 +78,20 @@ static db_cmdfcn_t db_watchdog; */ static struct command db_show_all_cmds[] = { - { (char *)0 } -}; - -static struct command_table db_show_all_table = { - db_show_all_cmds, - SET_BEGIN(db_show_all_cmd_set), - SET_LIMIT(db_show_all_cmd_set) + { "trace", db_stack_trace_all, 0, 0 }, }; +struct command_table db_show_all_table = + LIST_HEAD_INITIALIZER(db_show_all_table); static struct command db_show_cmds[] = { { "all", 0, 0, &db_show_all_table }, { "registers", db_show_regs, 0, 0 }, { "breaks", db_listbreak_cmd, 0, 0 }, { "threads", db_show_threads, 0, 0 }, - { (char *)0, } }; +struct command_table db_show_table = LIST_HEAD_INITIALIZER(db_show_table); -static struct command_table db_show_table = { - db_show_cmds, - SET_BEGIN(db_show_cmd_set), - SET_LIMIT(db_show_cmd_set) -}; - -static struct command db_commands[] = { +static struct command db_cmds[] = { { "print", db_print_cmd, 0, 0 }, { "p", db_print_cmd, 0, 0 }, { "examine", db_examine_cmd, CS_SET_DOT, 0 }, @@ -130,6 +117,7 @@ static struct command db_commands[] = { { "match", db_trace_until_matching_cmd,0, 0 }, { "trace", db_stack_trace, CS_OWN, 0 }, { "t", db_stack_trace, CS_OWN, 0 }, + /* XXX alias for all trace */ { "alltrace", db_stack_trace_all, 0, 0 }, { "where", db_stack_trace, CS_OWN, 0 }, { "bt", db_stack_trace, CS_OWN, 0 }, @@ -149,14 +137,8 @@ static struct command db_commands[] = { { "unscript", db_unscript_cmd, CS_OWN, 0 }, { "capture", db_capture_cmd, CS_OWN, 0 }, { "textdump", db_textdump_cmd, CS_OWN, 0 }, - { (char *)0, } -}; - -static struct command_table db_command_table = { - db_commands, - SET_BEGIN(db_cmd_set), - SET_LIMIT(db_cmd_set) }; +struct command_table db_cmd_table = LIST_HEAD_INITIALIZER(db_cmd_table); static struct command *db_last_command = 0; @@ -197,6 +179,72 @@ static void db_command(struct command ** struct command_table *cmd_table, int dopager); /* + * Initialize the command lists from the static tables. + */ +void +db_command_init(void) +{ +#define N(a) (sizeof(a) / sizeof(a[0])) + int i; + + for (i = 0; i < N(db_cmds); i++) + db_command_register(&db_cmd_table, &db_cmds[i]); + for (i = 0; i < N(db_show_cmds); i++) + db_command_register(&db_show_table, &db_show_cmds[i]); + for (i = 0; i < N(db_show_all_cmds); i++) + db_command_register(&db_show_all_table, &db_show_all_cmds[i]); +#undef N +} + +/* + * Register a command. + */ +void +db_command_register(struct command_table *list, struct command *cmd) +{ + struct command *c, *last; + + last = NULL; + LIST_FOREACH(c, list, next) { + int n = strcmp(cmd->name, c->name); + + /* Check that the command is not already present. */ + if (n == 0) { + printf("%s: Warning, the command \"%s\" already exists;" + " ignoring request\n", __func__, cmd->name); + return; + } + if (n < 0) { + /* NB: keep list sorted lexicographically */ + LIST_INSERT_BEFORE(c, cmd, next); + return; + } + last = c; + } + if (last == NULL) + LIST_INSERT_HEAD(list, cmd, next); + else + LIST_INSERT_AFTER(last, cmd, next); +} + +/* + * Remove a command previously registered with db_command_register. + */ +void +db_command_unregister(struct command_table *list, struct command *cmd) +{ + struct command *c; + + LIST_FOREACH(c, list, next) { + if (cmd == c) { + LIST_REMOVE(cmd, next); + return; + } + } + /* NB: intentionally quiet */ +} + +/* * Helper function to match a single command. */ static void @@ -245,22 +293,14 @@ db_cmd_search(name, table, cmdp) struct command **cmdp; /* out */ { struct command *cmd; - struct command **aux_cmdp; int result = CMD_NONE; - for (cmd = table->table; cmd->name != 0; cmd++) { - db_cmd_match(name, cmd, cmdp, &result); + LIST_FOREACH(cmd, table, next) { + db_cmd_match(name,cmd,cmdp,&result); if (result == CMD_UNIQUE) - return (CMD_UNIQUE); + break; } - if (table->aux_tablep != NULL) - for (aux_cmdp = table->aux_tablep; - aux_cmdp < table->aux_tablep_end; - aux_cmdp++) { - db_cmd_match(name, *aux_cmdp, cmdp, &result); - if (result == CMD_UNIQUE) - return (CMD_UNIQUE); - } + if (result == CMD_NONE) { /* check for 'help' */ if (name[0] == 'h' && name[1] == 'e' @@ -274,19 +314,11 @@ static void db_cmd_list(table) struct command_table *table; { - register struct command *cmd; - register struct command **aux_cmdp; + register struct command *cmd; - for (cmd = table->table; cmd->name != 0; cmd++) { - db_printf("%-12s", cmd->name); - db_end_line(12); - } - if (table->aux_tablep == NULL) - return; - for (aux_cmdp = table->aux_tablep; aux_cmdp < table->aux_tablep_end; - aux_cmdp++) { - db_printf("%-12s", (*aux_cmdp)->name); - db_end_line(12); + LIST_FOREACH(cmd, table, next) { + db_printf("%-12s", cmd->name); + db_end_line(12); } } @@ -296,7 +328,7 @@ db_command(last_cmdp, cmd_table, dopager struct command_table *cmd_table; int dopager; { - struct command *cmd; + struct command *cmd = NULL; int t; char modif[TOK_STRING_SIZE]; db_expr_t addr, count; @@ -463,7 +495,7 @@ db_command_loop() db_printf("db> "); (void) db_read_line(); - db_command(&db_last_command, &db_command_table, /* dopager */ 1); + db_command(&db_last_command, &db_cmd_table, /* dopager */ 1); } } @@ -481,7 +513,7 @@ db_command_script(const char *command) { db_prev = db_next = db_dot; db_inject_line(command); - db_command(&db_last_command, &db_command_table, /* dopager */ 0); + db_command(&db_last_command, &db_cmd_table, /* dopager */ 0); } void Modified: user/kmacy/releng_7_2_fcs/sys/ddb/db_command.h ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/ddb/db_command.h Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/ddb/db_command.h Wed May 20 07:31:11 2009 (r192431) @@ -37,6 +37,7 @@ * Command loop declarations. */ +void db_command_init(void); void db_command_loop(void); void db_command_script(const char *command); Modified: user/kmacy/releng_7_2_fcs/sys/ddb/db_main.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/ddb/db_main.c Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/ddb/db_main.c Wed May 20 07:31:11 2009 (r192431) @@ -172,6 +172,7 @@ db_init(void) uintptr_t symtab, strtab; Elf_Size tabsz, strsz; + db_command_init(); if (ksym_end > ksym_start && ksym_start != 0) { symtab = ksym_start; tabsz = *((Elf_Size*)symtab); Modified: user/kmacy/releng_7_2_fcs/sys/ddb/ddb.h ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/ddb/ddb.h Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/ddb/ddb.h Wed May 20 07:31:11 2009 (r192431) @@ -43,6 +43,9 @@ SYSCTL_DECL(_debug_ddb); #include /* type definitions */ +#include /* LIST_* */ +#include /* SYSINIT */ + #ifndef DB_MAXARGS #define DB_MAXARGS 10 #endif @@ -73,36 +76,97 @@ SYSCTL_DECL(_debug_ddb); int DB_CALL(db_expr_t, db_expr_t *, int, db_expr_t[]); #endif +/* + * There are three "command tables": + * - One for simple commands; a list of these is displayed + * by typing 'help' at the debugger prompt. + * - One for sub-commands of 'show'; to see this type 'show' + * without any arguments. + * - The last one for sub-commands of 'show all'; type 'show all' + * without any argument to get a list. + */ +struct command; +LIST_HEAD(command_table, command); +extern struct command_table db_cmd_table; +extern struct command_table db_show_table; +extern struct command_table db_show_all_table; + +/* + * Type signature for a function implementing a ddb command. + */ typedef void db_cmdfcn_t(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif); -#define DB_COMMAND(cmd_name, func_name) \ - DB_FUNC(cmd_name, func_name, db_cmd_set, 0, NULL) -#define DB_SHOW_COMMAND(cmd_name, func_name) \ - DB_FUNC(cmd_name, func_name, db_show_cmd_set, 0, NULL) -#define DB_SHOW_ALL_COMMAND(cmd_name, func_name) \ - DB_FUNC(cmd_name, func_name, db_show_all_cmd_set, 0, NULL) +/* + * Command table entry. + */ +struct command { + char * name; /* command name */ + db_cmdfcn_t *fcn; /* function to call */ + int flag; /* extra info: */ +#define CS_OWN 0x1 /* non-standard syntax */ +#define CS_MORE 0x2 /* standard syntax, but may have other words + * at end */ +#define CS_SET_DOT 0x100 /* set dot after command */ + struct command_table *more; /* another level of command */ + LIST_ENTRY(command) next; /* next entry in the command table */ +}; -#define DB_SET(cmd_name, func_name, set, flag, more) \ -static const struct command __CONCAT(cmd_name,_cmd) = { \ - __STRING(cmd_name), \ - func_name, \ - flag, \ - more \ +/* + * Arrange for the specified ddb command to be defined and + * bound to the specified function. Commands can be defined + * in modules in which case they will be available only when + * the module is loaded. + */ +#define _DB_SET(_suffix, _name, _func, list, _flag, _more) \ +static struct command __CONCAT(_name,_suffix) = { \ + .name = __STRING(_name), \ + .fcn = _func, \ + .flag = _flag, \ + .more = _more \ }; \ -TEXT_SET(set, __CONCAT(cmd_name,_cmd)) +static void __CONCAT(__CONCAT(_name,_suffix),_add)(void *arg __unused) \ + { db_command_register(&list, &__CONCAT(_name,_suffix)); } \ +SYSINIT(__CONCAT(_name,_suffix), SI_SUB_KLD, SI_ORDER_ANY, \ + __CONCAT(__CONCAT(_name,_suffix),_add), NULL); \ +static void __CONCAT(__CONCAT(_name,_suffix),_del)(void *arg __unused) \ + { db_command_unregister(&list, &__CONCAT(_name,_suffix)); } \ +SYSUNINIT(__CONCAT(_name,_suffix), SI_SUB_KLD, SI_ORDER_ANY, \ + __CONCAT(__CONCAT(_name,_suffix),_del), NULL); -#define DB_FUNC(cmd_name, func_name, set, flag, more) \ -static db_cmdfcn_t func_name; \ - \ -DB_SET(cmd_name, func_name, set, flag, more); \ - \ +/* + * Like _DB_SET but also create the function declaration which + * must be followed immediately by the body; e.g. + * _DB_FUNC(_cmd, panic, db_panic, db_cmd_table, 0, NULL) + * { + * ...panic implementation... + * } + * + * This macro is mostly used to define commands placed in one of + * the ddb command tables; see DB_COMMAND, etc. below. + */ +#define _DB_FUNC(_suffix, _name, _func, list, _flag, _more) \ +static db_cmdfcn_t _func; \ +_DB_SET(_suffix, _name, _func, list, _flag, _more); \ static void \ -func_name(addr, have_addr, count, modif) \ - db_expr_t addr; \ - boolean_t have_addr; \ - db_expr_t count; \ - char *modif; +_func(db_expr_t addr, boolean_t have_addr, db_expr_t count, char *modif) + +/* common idom provided for backwards compatibility */ +#define DB_FUNC(_name, _func, list, _flag, _more) \ + _DB_FUNC(_cmd, _name, _func, list, _flag, _more) + +#define DB_COMMAND(cmd_name, func_name) \ + _DB_FUNC(_cmd, cmd_name, func_name, db_cmd_table, 0, NULL) +#define DB_ALIAS(alias_name, func_name) \ + _DB_SET(_cmd, alias_name, func_name, db_cmd_table, 0, NULL) +#define DB_SHOW_COMMAND(cmd_name, func_name) \ + _DB_FUNC(_show, cmd_name, func_name, db_show_table, 0, NULL) +#define DB_SHOW_ALIAS(alias_name, func_name) \ + _DB_SET(_show, alias_name, func_name, db_show_table, 0, NULL) +#define DB_SHOW_ALL_COMMAND(cmd_name, func_name) \ + _DB_FUNC(_show_all, cmd_name, func_name, db_show_all_table, 0, NULL) +#define DB_SHOW_ALL_ALIAS(alias_name, func_name) \ + _DB_SET(_show_all, alias_name, func_name, db_show_all_table, 0, NULL) extern db_expr_t db_maxoff; extern int db_indent; @@ -150,6 +214,8 @@ void db_trace_self(void); int db_trace_thread(struct thread *, int); int db_value_of_name(const char *name, db_expr_t *valuep); int db_write_bytes(vm_offset_t addr, size_t size, char *data); +void db_command_register(struct command_table *, struct command *); +void db_command_unregister(struct command_table *, struct command *); db_cmdfcn_t db_breakpoint_cmd; db_cmdfcn_t db_capture_cmd; @@ -179,28 +245,6 @@ db_cmdfcn_t db_watchpoint_cmd; db_cmdfcn_t db_write_cmd; /* - * Command table. - */ -struct command; - -struct command_table { - struct command *table; - struct command **aux_tablep; - struct command **aux_tablep_end; -}; - -struct command { - char * name; /* command name */ - db_cmdfcn_t *fcn; /* function to call */ - int flag; /* extra info: */ -#define CS_OWN 0x1 /* non-standard syntax */ -#define CS_MORE 0x2 /* standard syntax, but may have other words - * at end */ -#define CS_SET_DOT 0x100 /* set dot after command */ - struct command_table *more; /* another level of command */ -}; - -/* * Interface between DDB and the DDB output capture facility. */ struct dumperinfo; Modified: user/kmacy/releng_7_2_fcs/sys/kern/kern_mutex.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/kern/kern_mutex.c Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/kern/kern_mutex.c Wed May 20 07:31:11 2009 (r192431) @@ -84,6 +84,7 @@ __FBSDID("$FreeBSD$"); #define mtx_owner(m) ((struct thread *)((m)->mtx_lock & ~MTX_FLAGMASK)) +static void assert_mtx(struct lock_object *lock, int what); #ifdef DDB static void db_show_mtx(struct lock_object *lock); #endif @@ -98,6 +99,7 @@ static int unlock_spin(struct lock_objec struct lock_class lock_class_mtx_sleep = { .lc_name = "sleep mutex", .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, + .lc_assert = assert_mtx, #ifdef DDB .lc_ddb_show = db_show_mtx, #endif @@ -107,6 +109,7 @@ struct lock_class lock_class_mtx_sleep = struct lock_class lock_class_mtx_spin = { .lc_name = "spin mutex", .lc_flags = LC_SPINLOCK | LC_RECURSABLE, + .lc_assert = assert_mtx, #ifdef DDB .lc_ddb_show = db_show_mtx, #endif @@ -120,19 +123,12 @@ struct lock_class lock_class_mtx_spin = struct mtx blocked_lock; struct mtx Giant; -#ifdef LOCK_PROFILING -static inline void lock_profile_init(void) +void +assert_mtx(struct lock_object *lock, int what) { - int i; - /* Initialize the mutex profiling locks */ - for (i = 0; i < LPROF_LOCK_SIZE; i++) { - mtx_init(&lprof_locks[i], "mprof lock", - NULL, MTX_SPIN|MTX_QUIET|MTX_NOPROFILE); - } + + mtx_assert((struct mtx *)lock, what); } -#else -static inline void lock_profile_init(void) {;} -#endif void lock_mtx(struct lock_object *lock, int how) @@ -181,7 +177,7 @@ _mtx_lock_flags(struct mtx *m, int opts, ("mtx_lock() of spin mutex %s @ %s:%d", m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, - file, line); + file, line, NULL); _get_sleep_lock(m, curthread, opts, file, line); LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, @@ -225,7 +221,7 @@ _mtx_lock_spin_flags(struct mtx *m, int ("mtx_lock_spin: recursed on non-recursive mutex %s @ %s:%d\n", m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, opts | LOP_NEWORDER | LOP_EXCLUSIVE, - file, line); + file, line, NULL); _get_spin_lock(m, curthread, opts, file, line); LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, line); @@ -258,9 +254,12 @@ _mtx_unlock_spin_flags(struct mtx *m, in int _mtx_trylock(struct mtx *m, int opts, const char *file, int line) { - int rval, contested = 0; +#ifdef LOCK_PROFILING uint64_t waittime = 0; - + int contested = 0; +#endif + int rval; + MPASS(curthread != NULL); KASSERT(m->mtx_lock != MTX_DESTROYED, ("mtx_trylock() of destroyed mutex @ %s:%d", file, line)); @@ -300,16 +299,18 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t int line) { struct turnstile *ts; + uintptr_t v; #ifdef ADAPTIVE_MUTEXES volatile struct thread *owner; #endif #ifdef KTR int cont_logged = 0; #endif +#ifdef LOCK_PROFILING int contested = 0; uint64_t waittime = 0; - uintptr_t v; - +#endif + if (mtx_owned(m)) { KASSERT((m->lock_object.lo_flags & LO_RECURSABLE) != 0, ("_mtx_lock_sleep: recursed on non-recursive mutex %s @ %s:%d\n", @@ -328,7 +329,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t "_mtx_lock_sleep: %s contested (lock=%p) at %s:%d", m->lock_object.lo_name, (void *)m->mtx_lock, file, line); - while (!_obtain_lock(m, tid)) { + while (!_obtain_lock(m, tid)) { #ifdef ADAPTIVE_MUTEXES /* * If the owner is running on another CPU, spin until the @@ -337,11 +338,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t v = m->mtx_lock; if (v != MTX_UNOWNED) { owner = (struct thread *)(v & ~MTX_FLAGMASK); -#ifdef ADAPTIVE_GIANT if (TD_IS_RUNNING(owner)) { -#else - if (m != &Giant && TD_IS_RUNNING(owner)) { -#endif if (LOCK_LOG_TEST(&m->lock_object, 0)) CTR3(KTR_LOCK, "%s: spinning on %p held by %p", @@ -375,11 +372,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t * CPU quit the hard path and try to spin. */ owner = (struct thread *)(v & ~MTX_FLAGMASK); -#ifdef ADAPTIVE_GIANT if (TD_IS_RUNNING(owner)) { -#else - if (m != &Giant && TD_IS_RUNNING(owner)) { -#endif turnstile_cancel(ts); cpu_spinwait(); continue; @@ -426,8 +419,8 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t m->lock_object.lo_name, (void *)tid, file, line); } #endif - lock_profile_obtain_lock_success(&m->lock_object, contested, - waittime, (file), (line)); + lock_profile_obtain_lock_success(&m->lock_object, contested, + waittime, file, line); } static void @@ -460,9 +453,12 @@ void _mtx_lock_spin(struct mtx *m, uintptr_t tid, int opts, const char *file, int line) { - int i = 0, contested = 0; + int i = 0; +#ifdef LOCK_PROFILING + int contested = 0; uint64_t waittime = 0; - +#endif + if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spinning", m); @@ -488,7 +484,7 @@ _mtx_lock_spin(struct mtx *m, uintptr_t if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); - lock_profile_obtain_lock_success(&m->lock_object, contested, + lock_profile_obtain_lock_success(&m->lock_object, contested, waittime, (file), (line)); } #endif /* SMP */ @@ -498,11 +494,13 @@ _thread_lock_flags(struct thread *td, in { struct mtx *m; uintptr_t tid; - int i, contested; - uint64_t waittime; + int i; +#ifdef LOCK_PROFILING + int contested = 0; + uint64_t waittime = 0; +#endif - contested = i = 0; - waittime = 0; + i = 0; tid = (uintptr_t)curthread; for (;;) { retry: @@ -518,13 +516,14 @@ retry: ("thread_lock: recursed on non-recursive mutex %s @ %s:%d\n", m->lock_object.lo_name, file, line)); WITNESS_CHECKORDER(&m->lock_object, - opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line); + opts | LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); while (!_obtain_lock(m, tid)) { if (m->mtx_lock == tid) { m->mtx_recurse++; break; } - lock_profile_obtain_lock_failed(&m->lock_object, &contested, &waittime); + lock_profile_obtain_lock_failed(&m->lock_object, + &contested, &waittime); /* Give interrupts a chance while we spin. */ spinlock_exit(); while (m->mtx_lock != MTX_UNOWNED) { @@ -545,8 +544,9 @@ retry: break; _rel_spin_lock(m); /* does spinlock_exit() */ } - lock_profile_obtain_lock_success(&m->lock_object, contested, - waittime, (file), (line)); + if (m->mtx_recurse == 0) + lock_profile_obtain_lock_success(&m->lock_object, contested, + waittime, (file), (line)); LOCK_LOG_LOCK("LOCK", &m->lock_object, opts, m->mtx_recurse, file, line); WITNESS_LOCK(&m->lock_object, opts | LOP_EXCLUSIVE, file, line); @@ -614,10 +614,10 @@ _mtx_unlock_sleep(struct mtx *m, int opt ts = turnstile_lookup(&m->lock_object); if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); - MPASS(ts != NULL); turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); _release_lock_quick(m); + /* * This turnstile is now no longer associated with the mutex. We can * unlock the chain lock so a new turnstile may take it's place. @@ -807,8 +807,6 @@ mutex_init(void) mtx_init(&proc0.p_slock, "process slock", NULL, MTX_SPIN | MTX_RECURSE); mtx_init(&devmtx, "cdev", NULL, MTX_DEF); mtx_lock(&Giant); - - lock_profile_init(); } #ifdef DDB @@ -846,7 +844,7 @@ db_show_mtx(struct lock_object *lock) if (!mtx_unowned(m) && !mtx_destroyed(m)) { td = mtx_owner(m); db_printf(" owner: %p (tid %d, pid %d, \"%s\")\n", td, - td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); + td->td_tid, td->td_proc->p_pid, td->td_name); if (mtx_recursed(m)) db_printf(" recursed: %d\n", m->mtx_recurse); } Modified: user/kmacy/releng_7_2_fcs/sys/kern/kern_rmlock.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/kern/kern_rmlock.c Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/kern/kern_rmlock.c Wed May 20 07:31:11 2009 (r192431) @@ -32,7 +32,7 @@ */ #include -__FBSDID("$FreeBSD$"); +__FBSDID("$FreeBSD: head/sys/kern/kern_rmlock.c 191539 2009-04-26 21:16:03Z rwatson $"); #include "opt_ddb.h" @@ -69,12 +69,14 @@ static __inline void compiler_memory_bar __asm __volatile("":::"memory"); } +static void assert_rm(struct lock_object *lock, int what); static void lock_rm(struct lock_object *lock, int how); static int unlock_rm(struct lock_object *lock); struct lock_class lock_class_rm = { .lc_name = "rm", .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE, + .lc_assert = assert_rm, #if 0 #ifdef DDB .lc_ddb_show = db_show_rwlock, @@ -85,6 +87,13 @@ struct lock_class lock_class_rm = { }; static void +assert_rm(struct lock_object *lock, int what) +{ + + panic("assert_rm called"); +} + +static void lock_rm(struct lock_object *lock, int how) { @@ -403,7 +412,7 @@ void _rm_wlock_debug(struct rmlock *rm, { WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, - file, line); + file, line, NULL); _rm_wlock(rm); @@ -430,7 +439,7 @@ _rm_rlock_debug(struct rmlock *rm, struc const char *file, int line) { - WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line); + WITNESS_CHECKORDER(&rm->lock_object, LOP_NEWORDER, file, line, NULL); _rm_rlock(rm, tracker); Modified: user/kmacy/releng_7_2_fcs/sys/kern/kern_rwlock.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/kern/kern_rwlock.c Wed May 20 07:11:46 2009 (r192430) +++ user/kmacy/releng_7_2_fcs/sys/kern/kern_rwlock.c Wed May 20 07:31:11 2009 (r192431) @@ -39,10 +39,12 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include +#include #include #include @@ -54,17 +56,27 @@ CTASSERT((RW_RECURSE & LO_CLASSFLAGS) == #define ADAPTIVE_RWLOCKS #endif +#ifdef ADAPTIVE_RWLOCKS +static int rowner_retries = 10; +static int rowner_loops = 10000; +SYSCTL_NODE(_debug, OID_AUTO, rwlock, CTLFLAG_RD, NULL, "rwlock debugging"); +SYSCTL_INT(_debug_rwlock, OID_AUTO, retry, CTLFLAG_RW, &rowner_retries, 0, ""); +SYSCTL_INT(_debug_rwlock, OID_AUTO, loops, CTLFLAG_RW, &rowner_loops, 0, ""); +#endif + #ifdef DDB #include static void db_show_rwlock(struct lock_object *lock); #endif +static void assert_rw(struct lock_object *lock, int what); static void lock_rw(struct lock_object *lock, int how); static int unlock_rw(struct lock_object *lock); struct lock_class lock_class_rw = { .lc_name = "rw", .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE, + .lc_assert = assert_rw, #ifdef DDB .lc_ddb_show = db_show_rwlock, #endif @@ -103,6 +115,13 @@ struct lock_class lock_class_rw = { #endif void +assert_rw(struct lock_object *lock, int what) +{ + + rw_assert((struct rwlock *)lock, what); +} + +void lock_rw(struct lock_object *lock, int how) { struct rwlock *rw; @@ -172,6 +191,14 @@ rw_sysinit(void *arg) rw_init(args->ra_rw, args->ra_desc); } +void +rw_sysinit_flags(void *arg) +{ + struct rw_args_flags *args = arg; + + rw_init_flags(args->ra_rw, args->ra_desc, args->ra_flags); +} + int rw_wowned(struct rwlock *rw) { @@ -187,7 +214,7 @@ _rw_wlock(struct rwlock *rw, const char KASSERT(rw->rw_lock != RW_DESTROYED, ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, - line); + line, NULL); __rw_wlock(rw, curthread, file, line); LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); @@ -234,6 +261,17 @@ _rw_wunlock(struct rwlock *rw, const cha lock_profile_release_lock(&rw->lock_object); __rw_wunlock(rw, curthread, file, line); } +/* + * Determines whether a new reader can acquire a lock. Succeeds if the + * reader already owns a read lock and the lock is locked for read to + * prevent deadlock from reader recursion. Also succeeds if the lock + * is unlocked and has no writer waiters or spinners. Failing otherwise + * prioritizes writers before readers. + */ +#define RW_CAN_READ(_rw) \ + ((curthread->td_rw_rlocks && (_rw) & RW_LOCK_READ) || ((_rw) & \ + (RW_LOCK_READ | RW_LOCK_WRITE_WAITERS | RW_LOCK_WRITE_SPINNER)) == \ + RW_LOCK_READ) void _rw_rlock(struct rwlock *rw, const char *file, int line) @@ -241,31 +279,22 @@ _rw_rlock(struct rwlock *rw, const char struct turnstile *ts; #ifdef ADAPTIVE_RWLOCKS volatile struct thread *owner; + int spintries = 0; + int i; #endif -#ifdef LOCK_PROFILING_SHARED +#ifdef LOCK_PROFILING uint64_t waittime = 0; int contested = 0; #endif - uintptr_t x; + uintptr_t v; KASSERT(rw->rw_lock != RW_DESTROYED, ("rw_rlock() of destroyed rwlock @ %s:%d", file, line)); KASSERT(rw_wowner(rw) != curthread, ("%s (%s): wlock already held @ %s:%d", __func__, rw->lock_object.lo_name, file, line)); - WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line); + WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER, file, line, NULL); - /* - * Note that we don't make any attempt to try to block read - * locks once a writer has blocked on the lock. The reason is - * that we currently allow for read locks to recurse and we - * don't keep track of all the holders of read locks. Thus, if - * we were to block readers once a writer blocked and a reader - * tried to recurse on their reader lock after a writer had - * blocked we would end up in a deadlock since the reader would - * be blocked on the writer, and the writer would be blocked - * waiting for the reader to release its original read lock. - */ for (;;) { /* * Handle the easy case. If no other thread has a write @@ -277,33 +306,27 @@ _rw_rlock(struct rwlock *rw, const char * completely unlocked rwlock since such a lock is encoded * as a read lock with no waiters. */ - x = rw->rw_lock; - if (x & RW_LOCK_READ) { - + v = rw->rw_lock; + if (RW_CAN_READ(v)) { /* * The RW_LOCK_READ_WAITERS flag should only be set - * if another thread currently holds a write lock, - * and in that case RW_LOCK_READ should be clear. + * if the lock has been unlocked and write waiters + * were present. */ - MPASS((x & RW_LOCK_READ_WAITERS) == 0); - if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, - x + RW_ONE_READER)) { -#ifdef LOCK_PROFILING_SHARED - if (RW_READERS(x) == 0) - lock_profile_obtain_lock_success( - &rw->lock_object, contested, - waittime, file, line); -#endif + if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, + v + RW_ONE_READER)) { if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR4(KTR_LOCK, "%s: %p succeed %p -> %p", __func__, - rw, (void *)x, - (void *)(x + RW_ONE_READER)); + rw, (void *)v, + (void *)(v + RW_ONE_READER)); break; } cpu_spinwait(); continue; } + lock_profile_obtain_lock_failed(&rw->lock_object, + &contested, &waittime); #ifdef ADAPTIVE_RWLOCKS /* @@ -311,36 +334,45 @@ _rw_rlock(struct rwlock *rw, const char * the owner stops running or the state of the lock * changes. */ - owner = (struct thread *)RW_OWNER(x); - if (TD_IS_RUNNING(owner)) { - if (LOCK_LOG_TEST(&rw->lock_object, 0)) - CTR3(KTR_LOCK, "%s: spinning on %p held by %p", - __func__, rw, owner); -#ifdef LOCK_PROFILING_SHARED - lock_profile_obtain_lock_failed(&rw->lock_object, - &contested, &waittime); -#endif - while ((struct thread*)RW_OWNER(rw->rw_lock) == owner && - TD_IS_RUNNING(owner)) + if ((v & RW_LOCK_READ) == 0) { + owner = (struct thread *)RW_OWNER(v); + if (TD_IS_RUNNING(owner)) { + if (LOCK_LOG_TEST(&rw->lock_object, 0)) + CTR3(KTR_LOCK, + "%s: spinning on %p held by %p", + __func__, rw, owner); + while ((struct thread*)RW_OWNER(rw->rw_lock) == + owner && TD_IS_RUNNING(owner)) + cpu_spinwait(); + continue; + } + } else if (spintries < rowner_retries) { + spintries++; + for (i = 0; i < rowner_loops; i++) { + v = rw->rw_lock; + if ((v & RW_LOCK_READ) == 0 || RW_CAN_READ(v)) + break; cpu_spinwait(); - continue; + } + if (i != rowner_loops) + continue; } #endif /* * Okay, now it's the hard case. Some other thread already - * has a write lock, so acquire the turnstile lock so we can - * begin the process of blocking. + * has a write lock or there are write waiters present, + * acquire the turnstile lock so we can begin the process + * of blocking. */ ts = turnstile_trywait(&rw->lock_object); /* * The lock might have been released while we spun, so - * recheck its state and restart the loop if there is no - * longer a write lock. + * recheck its state and restart the loop if needed. */ - x = rw->rw_lock; - if (x & RW_LOCK_READ) { + v = rw->rw_lock; + if (RW_CAN_READ(v)) { turnstile_cancel(ts); cpu_spinwait(); continue; @@ -351,23 +383,30 @@ _rw_rlock(struct rwlock *rw, const char * If the current owner of the lock is executing on another * CPU quit the hard path and try to spin. */ - owner = (struct thread *)RW_OWNER(x); - if (TD_IS_RUNNING(owner)) { - turnstile_cancel(ts); - cpu_spinwait(); - continue; + if ((v & RW_LOCK_READ) == 0) { + owner = (struct thread *)RW_OWNER(v); + if (TD_IS_RUNNING(owner)) { + turnstile_cancel(ts); + cpu_spinwait(); + continue; + } } #endif /* - * Ok, it's still a write lock. If the RW_LOCK_READ_WAITERS - * flag is already set, then we can go ahead and block. If - * it is not set then try to set it. If we fail to set it - * drop the turnstile lock and restart the loop. - */ - if (!(x & RW_LOCK_READ_WAITERS)) { *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***