Reenable nss_db with a completely new implementation

No longer is Berkeley db used.  Instead a simple hash function is used.
The database files are not updated once they are created and therefore
no complicated database is needed.

24 files changed, 555 insertions, 1391 deletions

diff --git a/ChangeLog b/ChangeLog
index 9ce38a8516..60ad9ae1f0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,35 @@
+2011-06-14  Ulrich Drepper  <drepper@gmail.com>
+
+	* Versions.def: Add entry for libnss_db.
+	* shlib-versions: Likewise.
+	* nss/Makefile: Add rules to build libnss_db.
+	* nss/Versions: Add libnss_db information.  Organize libnss_files
+	entries better.
+	* nss/db-Makefile: Add gshadow support.  Change rules for the new
+	makedb progra.  Some minor improvements to generate smaller files.
+	* nss/nss_db/nss_db.h: Move NSS database header data structures to
+	here from...
+	* nss/makedb.c: ...here.
+	Improve database format to be smaller and require less memory at
+	runtime.
+	* nss/nss_db/db-XXX.x: Adjust for new database format.  Don't use
+	db anymore.
+	* nss/nss_db/db-netgrp.c: Likewise.
+	* nss/nss_db/db-open.c: Likewise.
+	* nss/nss_files/flies-XXX.x: Adjust comments.
+	* nss/nss_files/files-ethers.c: Adjust for new DB_LOOKUP definition.
+	* nss/nss_files/files-grp.c: Likewise.
+	* nss/nss_files/files-hosts.c: Likewise.
+	* nss/nss_files/files-network.c: Likewise.
+	* nss/nss_files/files-proto.c: Likewise.
+	* nss/nss_files/files-pwd.c: Likewise.
+	* nss/nss_files/files-rpc.c: Likewise.
+	* nss/nss_files/files-service.c: Likewise.
+	* nss/nss_files/files-sgrp.c: Likewise.
+	* nss/nss_files/files-spwd.c: Likewise.
+	* nss/nss_db/db-alias.c: Removed.
+	* nss/nss_db/dummy-db.h: Removed.
+
 2011-06-02  Ulrich Drepper  <drepper@gmail.com>
 
 	* nss/makedb.c: Rewritten to not use database library.
diff --git a/Versions.def b/Versions.def
index e478fdd9e5..666ea73108 100644
--- a/Versions.def
+++ b/Versions.def
@@ -67,6 +67,9 @@ libnss_compat {
 libnss_dns {
   GLIBC_PRIVATE
 }
+libnss_db {
+  GLIBC_PRIVATE
+}
 libnss_files {
   GLIBC_PRIVATE
 }
diff --git a/nss/Makefile b/nss/Makefile
index 16578a3548..a925cb5bf7 100644
--- a/nss/Makefile
+++ b/nss/Makefile
@@ -53,7 +53,7 @@ otherlibs += $(nssobjdir)/libnss_files.a $(resolvobjdir)/libnss_dns.a \
 endif
 
 # Specify rules for the nss_* modules.  We have some services.
-services		:= files
+services		:= files db
 
 extra-libs		= $(services:%=libnss_%)
 # These libraries will be built in the `others' pass rather than
@@ -69,10 +69,18 @@ libnss_files-routines	:= $(addprefix files-,$(databases)) \
 			   files-initgroups files-have_o_cloexec
 distribute		+= files-XXX.c files-parse.c
 
+libnss_db-dbs		:= $(addprefix db-,\
+				       $(filter-out hosts network key alias,\
+						    $(databases)))
+libnss_db-routines	:= $(libnss_db-dbs) db-open hash-string
+generated		+= $(filter-out db-alias.c db-netgrp.c, \
+					$(addsuffix .c,$(libnss_db-dbs)))
+distribute		+= $(addprefix nss_db/, db-XXX.c nss_db.h)
 
 # Build static module if requested
 ifneq ($(build-static-nss),yes)
 libnss_files-inhibit-o	= $(filter-out .os,$(object-suffixes))
+libnss_db-inhibit-o	= $(filter-out .os,$(object-suffixes))
 endif
 
 include ../Rules
@@ -93,6 +101,15 @@ libnss-libc = $(common-objpfx)linkobj/libc.so
 $(services:%=$(objpfx)libnss_%.so): $(libnss-libc) \
 				    $(common-objpfx)libc_nonshared.a
 
+$(objpfx)libnss_db.so: $(objpfx)libnss_files.so
+
+$(libnss_db-dbs:%=$(objpfx)%.c): $(objpfx)db-%.c: nss_files/files-%.c
+	@rm -f $@.new
+	(echo '#define EXTERN_PARSER';\
+	 echo '#define GENERIC "../nss_db/db-XXX.c"';\
+	 echo '#include "$<"') > $@.new
+	mv -f $@.new $@
+
 
 $(objpfx)makedb: $(makedb-modules:%=$(objpfx)%.o)
 
diff --git a/nss/Versions b/nss/Versions
index 1c6fd686f3..2f3a671af6 100644
--- a/nss/Versions
+++ b/nss/Versions
@@ -27,6 +27,8 @@ libnss_files {
     _nss_files_endetherent;
     _nss_files_getetherent_r;
     _nss_files_parse_etherent;
+    _nss_files_gethostton_r;
+    _nss_files_getntohost_r;
 
     _nss_files_setgrent;
     _nss_files_endgrent;
@@ -41,14 +43,12 @@ libnss_files {
     _nss_files_gethostbyname4_r;
     _nss_files_gethostbyname_r;
     _nss_files_gethostent_r;
-    _nss_files_gethostton_r;
 
     _nss_files_setnetent;
     _nss_files_endnetent;
     _nss_files_getnetbyaddr_r;
     _nss_files_getnetbyname_r;
     _nss_files_getnetent_r;
-    _nss_files_getntohost_r;
     _nss_files_parse_netent;
 
     _nss_files_setnetgrent;
@@ -99,3 +99,57 @@ libnss_files {
     _nss_files_initgroups_dyn;
   }
 }
+
+libnss_db {
+  GLIBC_PRIVATE {
+    _nss_db_setetherent;
+    _nss_db_endetherent;
+    _nss_db_getetherent_r;
+    _nss_db_gethostton_r;
+    _nss_db_getntohost_r;
+
+    _nss_db_setgrent;
+    _nss_db_endgrent;
+    _nss_db_getgrent_r;
+    _nss_db_getgrgid_r;
+    _nss_db_getgrnam_r;
+
+    _nss_db_setnetgrent;
+    _nss_db_endnetgrent;
+    _nss_db_getnetgrent_r;
+
+    _nss_db_setprotoent;
+    _nss_db_endprotoent;
+    _nss_db_getprotoent_r;
+    _nss_db_getprotobyname_r;
+    _nss_db_getprotobynumber_r;
+
+    _nss_db_setpwent;
+    _nss_db_endpwent;
+    _nss_db_getpwent_r;
+    _nss_db_getpwnam_r;
+    _nss_db_getpwuid_r;
+
+    _nss_db_setrpcent;
+    _nss_db_endrpcent;
+    _nss_db_getrpcent_r;
+    _nss_db_getrpcbyname_r;
+    _nss_db_getrpcbynumber_r;
+
+    _nss_db_setservent;
+    _nss_db_endservent;
+    _nss_db_getservent_r;
+    _nss_db_getservbyname_r;
+    _nss_db_getservbyport_r;
+
+    _nss_db_setsgent;
+    _nss_db_endsgent;
+    _nss_db_getsgent_r;
+    _nss_db_getsgnam_r;
+
+    _nss_db_setspent;
+    _nss_db_endspent;
+    _nss_db_getspent_r;
+    _nss_db_getspnam_r;
+  }
+}
diff --git a/nss/db-Makefile b/nss/db-Makefile
index f9c6bd37d7..649e09ced6 100644
--- a/nss/db-Makefile
+++ b/nss/db-Makefile
@@ -1,5 +1,5 @@
 # Makefile to (re-)generate db versions of system database files.
-# Copyright (C) 1996, 1997, 1998 Free Software Foundation, Inc.
+# Copyright (C) 1996, 1997, 1998, 2011 Free Software Foundation, Inc.
 # This file is part of the GNU C Library.
 # Contributed by Ulrich Drepper <drepper@cygnus.com>, 1996.
 #
@@ -20,7 +20,8 @@
 # 02111-1307 USA.
 
 DATABASES = $(wildcard /etc/passwd /etc/group /etc/ethers /etc/protocols \
-		       /etc/rpc /etc/services /etc/shadow /etc/netgroup)
+		       /etc/rpc /etc/services /etc/shadow /etc/gshadow \
+		       /etc/netgroup)
 
 VAR_DB = /var/db
 
@@ -32,10 +33,9 @@ all: $(patsubst %,$(VAR_DB)/%.db,$(notdir $(DATABASES)))
 
 $(VAR_DB)/passwd.db: /etc/passwd
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { FS=":"; OFS=":"; cnt=0 } \
+	@$(AWK) 'BEGIN { FS=":"; OFS=":" } \
 		 /^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print; \
 			   printf "=%s ", $$3; print }' $^ | \
 	$(MAKEDB) -o $@ -
@@ -43,10 +43,9 @@ $(VAR_DB)/passwd.db: /etc/passwd
 
 $(VAR_DB)/group.db: /etc/group
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { FS=":"; OFS=":"; cnt=0 } \
+	@$(AWK) 'BEGIN { FS=":"; OFS=":" } \
 		 /^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print; \
 			   printf "=%s ", $$3; print }' $^ | \
 	$(MAKEDB) -o $@ -
@@ -54,10 +53,8 @@ $(VAR_DB)/group.db: /etc/group
 
 $(VAR_DB)/ethers.db: /etc/ethers
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { cnt=0 } \
-		 /^[ \t]*$$/ { next } \
+	@$(AWK) '/^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print; \
 			   printf "=%s ", $$2; print }' $^ | \
 	$(MAKEDB) -o $@ -
@@ -65,10 +62,8 @@ $(VAR_DB)/ethers.db: /etc/ethers
 
 $(VAR_DB)/protocols.db: /etc/protocols
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { cnt=0 } \
-		 /^[ \t]*$$/ { next } \
+	@$(AWK) '/^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print; \
 			   printf "=%s ", $$2; print; \
 			   for (i = 3; i <= NF && !($$i ~ /^#/); ++i) \
@@ -78,10 +73,8 @@ $(VAR_DB)/protocols.db: /etc/protocols
 
 $(VAR_DB)/rpc.db: /etc/rpc
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { cnt=0 } \
-		 /^[ \t]*$$/ { next } \
+	@$(AWK) '/^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print; \
 			   printf "=%s ", $$2; print; \
 			   for (i = 3; i <= NF && !($$i ~ /^#/); ++i) \
@@ -91,26 +84,25 @@ $(VAR_DB)/rpc.db: /etc/rpc
 
 $(VAR_DB)/services.db: /etc/services
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { FS="[ \t/]+"; cnt=0 } \
+	@$(AWK) 'BEGIN { FS="[ \t/]+" } \
 		 /^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
-		 /^[^#]/ { printf ".%s/%s ", $$1, $$3; print; \
-			   printf ".%s/ ", $$1; print; \
+		 /^[^#]/ { sub(/[ \t]*#.*$$/, "");\
+			   printf ":%s/%s ", $$1, $$3; print; \
+			   printf ":%s/ ", $$1; print; \
 			   printf "=%s/%s ", $$2, $$3; print; \
 			   printf "=%s/ ", $$2; print; \
 			   for (i = 4; i <= NF && !($$i ~ /^#/); ++i) \
-			     { printf ".%s/%s ", $$i, $$3; print; \
-			       printf ".%s/ ", $$i; print } }' $^ | \
+			     { printf ":%s/%s ", $$i, $$3; print; \
+			       printf ":%s/ ", $$i; print } }' $^ | \
 	$(MAKEDB) -o $@ -
 	@echo "done."
 
 $(VAR_DB)/shadow.db: /etc/shadow
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { FS=":"; OFS=":"; cnt=0 } \
+	@$(AWK) 'BEGIN { FS=":"; OFS=":" } \
 		 /^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
 		 /^[^#]/ { printf ".%s ", $$1; print }' $^ | \
 	(umask 077 && $(MAKEDB) -o $@ -)
 	@echo "done."
@@ -126,14 +118,38 @@ $(VAR_DB)/shadow.db: /etc/shadow
 	  echo; \
 	fi
 
+$(VAR_DB)/gshadow.db: /etc/gshadow
+	@echo -n "$(patsubst %.db,%,$(@F))... "
+	@$(AWK) 'BEGIN { FS=":"; OFS=":" } \
+		 /^[ \t]*$$/ { next } \
+		 /^[ \t]*#/ { next } \
+		 /^[^#]/ { printf ".%s ", $$1; print }' $^ | \
+	(umask 077 && $(MAKEDB) -o $@ -)
+	@echo "done."
+	@if chgrp shadow $@ 2>/dev/null; then \
+	  chmod g+r $@; \
+	else \
+	  chown 0 $@; chgrp 0 $@; chmod 600 $@; \
+	  echo; \
+	  echo "Warning: The shadow group database $@"; \
+	  echo "has been set to be readable only by root.  You may want"; \
+	  echo "to make it readable by the \`shadow' group depending"; \
+	  echo "on your configuration."; \
+	  echo; \
+	fi
+
 $(VAR_DB)/netgroup.db: /etc/netgroup
 	@echo -n "$(patsubst %.db,%,$(@F))... "
-	@$(AWK) 'BEGIN { cnt=0 } \
+	@$(AWK) 'BEGIN { ini=1 } \
 		 /^[ \t]*$$/ { next } \
 		 /^[ \t]*#/ { next } \
-		 { printf "0%u ", cnt++; print } \
-		 /^[^#]/ { end=sub(/\\/, " "); \
+		 /^[^#]/ { if (sub(/[ \t]*\\$$/, " ") == 0) end="\n"; \
+			   else end=""; \
 			   gsub(/[ \t]+/, " "); \
-			   if(end == 1) printf "%s", $$0; else print }' $^ | \
+			   sub(/^[ \t]*/, ""); \
+			   if (ini == 0) printf "%s%s", $$0, end; \
+			   else printf ".%s %s%s", $$1, $$0, end; \
+			   ini=end == "" ? 0 : 1; } \
+			   END { if (ini==0) printf "\n" }' $^ | \
 	$(MAKEDB) -o $@ -
 	@echo "done."
diff --git a/nss/makedb.c b/nss/makedb.c
index a01b2350aa..687414b74f 100644
--- a/nss/makedb.c
+++ b/nss/makedb.c
@@ -34,7 +34,7 @@
 #include <unistd.h>
 #include <sys/mman.h>
 #include <sys/stat.h>
-#include "nss_db/dummy-db.h"
+#include "nss_db/nss_db.h"
 
 /* Get libc version number.  */
 #include "../version.h"
@@ -49,51 +49,21 @@
 
 #define PACKAGE _libc_intl_domainname
 
-/* String table index type.  */
-typedef uint32_t stridx_t;
-
-/* Database file header.  */
-struct nss_db_header
-{
-  uint32_t magic;
-#define NSS_DB_MAGIC 0xdd110601
-  uint32_t ndbs;
-  uint64_t valstroffset;
-  uint64_t valstrlen;
-  struct
-  {
-    char id;
-    enum nss_db_type { nss_db_type_hash = 0,
-		       nss_db_type_iterate,
-		       nss_db_type_int4 } type:8;
-    char pad[sizeof (uint32_t) - 2];
-    uint32_t hashsize;
-    uint64_t hashoffset;
-    uint64_t stroffset;
-  } dbs[0];
-};
-
-struct nss_db_entry
-{
-  stridx_t keyidx;
-  stridx_t dataidx;
-};
-
-
 /* List of data bases.  */
 struct database
 {
   char dbid;
-  enum nss_db_type type;
   struct database *next;
   void *entries;
   size_t nentries;
-  size_t keystrlen;
   size_t nhashentries;
-  struct nss_db_entry *hashtable;
+  stridx_t *hashtable;
+  size_t keystrlen;
+  stridx_t *keyidxtab;
   char *keystrtab;
 } *databases;
 static size_t ndatabases;
+static size_t nhashentries;
 static size_t valstrlen;
 static void *valstrtree;
 static char *valstrtab;
@@ -101,7 +71,6 @@ static char *valstrtab;
 /* Database entry.  */
 struct dbentry
 {
-  size_t keylen;
   stridx_t validx;
   uint32_t hashval;
   char str[0];
@@ -115,16 +84,6 @@ struct valstrentry
 };
 
 
-/* Database type specifiers.  */
-struct dbtype
-{
-  char dbid;
-  enum nss_db_type type;
-  struct dbtype *next;
-};
-static struct dbtype *dbtypes;
-
-
 /* True if any entry has been added.  */
 static bool any_dbentry;
 
@@ -154,10 +113,6 @@ static const struct argp_option options[] =
   { "undo", 'u', NULL, 0,
     N_("Print content of database file, one entry a line") },
   { NULL, 0, NULL, 0, N_("Select index type") },
-  { "iterate", 'I', "KEY", 0,
-    N_("Index identified by KEY used to iterate over database") },
-  { "binary", 'B', "KEY", 0,
-    N_("Index identified by KEY has binary key value") },
   { NULL, 0, NULL, 0, NULL }
 };
 
@@ -210,7 +165,7 @@ main (int argc, char *argv[])
   const char *input_name;
   FILE *input_file;
   int remaining;
-  int mode = 0666;
+  int mode = 0644;
 
   /* Set locale via LC_ALL.  */
   setlocale (LC_ALL, "");
@@ -292,7 +247,12 @@ main (int argc, char *argv[])
 
   /* Bail out if nothing is to be done.  */
   if (!any_dbentry)
-    error (EXIT_SUCCESS, 0, gettext ("no entries to be processed"));
+    {
+      if (be_quiet)
+	return EXIT_SUCCESS;
+      else
+	error (EXIT_SUCCESS, 0, gettext ("no entries to be processed"));
+    }
 
   /* Compute hash and string tables.  */
   compute_tables ();
@@ -308,7 +268,6 @@ main (int argc, char *argv[])
   reset_file_creation_context ();
   if (fd == -1)
     error (EXIT_FAILURE, errno, gettext ("cannot create temporary file"));
-  // XXX SELinux context
 
   status = write_output (fd);
 
@@ -352,7 +311,6 @@ main (int argc, char *argv[])
 static error_t
 parse_opt (int key, char *arg, struct argp_state *state)
 {
-  struct dbtype *newtype;
   switch (key)
     {
     case 'f':
@@ -367,17 +325,6 @@ parse_opt (int key, char *arg, struct argp_state *state)
     case 'u':
       do_undo = 1;
       break;
-    case 'I':
-    case 'B':
-      if (arg[0] == '\0' || arg[1] != '\0')
-	error (EXIT_FAILURE, 0, gettext ("\
-argument for option to specify database type must be a single-byte character"));
-      newtype = xmalloc (sizeof (struct dbtype));
-      newtype->dbid = arg[0];
-      newtype->type = key == 'I' ? nss_db_type_iterate : nss_db_type_int4;
-      newtype->next = dbtypes;
-      dbtypes = newtype;
-      break;
     default:
       return ARGP_ERR_UNKNOWN;
     }
@@ -424,13 +371,7 @@ dbentry_compare (const void *p1, const void *p2)
   if (d1->hashval != d2->hashval)
     return d1->hashval < d2->hashval ? -1 : 1;
 
-  if (d1->keylen < d2->keylen)
-    return -1;
-
-  if (d1->keylen > d2->keylen)
-    return 1;
-
-  return memcmp (d1->str, d2->str, d1->keylen);
+  return strcmp (d1->str, d2->str);
 }
 
 
@@ -522,22 +463,11 @@ process_input (input, inname, to_lowercase, be_quiet)
 	    {
 	      last_database = xmalloc (sizeof (*last_database));
 	      last_database->dbid = key[0];
-	      last_database->type = nss_db_type_hash;	/* Default.  */
 	      last_database->next = databases;
 	      last_database->entries = NULL;
 	      last_database->nentries = 0;
 	      last_database->keystrlen = 0;
 	      databases = last_database;
-
-	      struct dbtype *typeit = dbtypes;
-	      while (typeit != NULL)
-		if (typeit->dbid == last_database->dbid)
-		  {
-		    last_database->type = typeit->type;
-		    break;
-		  }
-		else
-		  typeit = typeit->next;
 	    }
 	}
 
@@ -545,24 +475,6 @@ process_input (input, inname, to_lowercase, be_quiet)
       ++key;
       --keylen;
 
-      /* Check the key value if it has to be numeric.  */
-      unsigned long int keyvalue = 0;
-      if (last_database->type != nss_db_type_hash)
-	{
-	  char *endp;
-	  errno = 0;
-	  keyvalue = strtoul (key, &endp, 0);
-	  if ((keyvalue == ULONG_MAX && errno == ERANGE)
-	      || keyvalue > ~((stridx_t) 0))
-	    error (EXIT_FAILURE, 0,
-		   gettext ("index value in line %zu too large"), linenr);
-
-	  if (*endp != '\0')
-	    error (EXIT_FAILURE, 0,
-		   gettext ("index value in line %zu is not a number"),
-		   linenr);
-	}
-
       /* Store the data.  */
       struct valstrentry *nentry = xmalloc (sizeof (struct valstrentry)
 					    + datalen);
@@ -584,24 +