diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/CMakeLists.txt ./heap/CMakeLists.txt --- D:\projects\mysql\mysql-5.0.45-orig/heap/CMakeLists.txt 2007-07-04 06:06:58.000000000 -0700 +++ ./heap/CMakeLists.txt 2008-02-15 12:47:52.000000000 -0800 @@ -20,4 +20,5 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/ ADD_LIBRARY(heap _check.c _rectest.c hp_block.c hp_clear.c hp_close.c hp_create.c hp_delete.c hp_extra.c hp_hash.c hp_info.c hp_open.c hp_panic.c hp_rename.c hp_rfirst.c hp_rkey.c hp_rlast.c hp_rnext.c hp_rprev.c + hp_dspace.c hp_record.c hp_rrnd.c hp_rsame.c hp_scan.c hp_static.c hp_update.c hp_write.c) diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/Makefile.am ./heap/Makefile.am --- D:\projects\mysql\mysql-5.0.45-orig/heap/Makefile.am 2007-07-04 06:05:36.000000000 -0700 +++ ./heap/Makefile.am 2008-02-15 12:48:56.000000000 -0800 @@ -25,6 +25,7 @@ libheap_a_SOURCES = hp_open.c hp_extra.c hp_rrnd.c hp_scan.c hp_update.c hp_write.c hp_delete.c \ hp_rsame.c hp_create.c hp_rename.c hp_rfirst.c \ hp_rnext.c hp_rlast.c hp_rprev.c hp_clear.c \ + hp_dspace.c hp_record.c \ hp_rkey.c hp_block.c \ hp_hash.c _check.c _rectest.c hp_static.c EXTRA_DIST = CMakeLists.txt diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/_check.c ./heap/_check.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/_check.c 2007-07-04 06:06:58.000000000 -0700 +++ ./heap/_check.c 2008-02-28 20:44:30.000000000 -0800 @@ -43,7 +43,7 @@ int heap_check_heap(HP_INFO *info, my_bo { int error; uint key; - ulong records=0, deleted=0, pos, next_block; + ulong records=0, deleted=0, chunk_count=0, pos, next_block; HP_SHARE *share=info->s; HP_INFO save_info= *info; /* Needed because scan_init */ DBUG_ENTER("heap_check_heap"); @@ -56,6 +56,7 @@ int heap_check_heap(HP_INFO *info, my_bo error|= check_one_key(share->keydef + key, key, share->records, share->blength, print_status); } + /* This is basicly the same code as in hp_scan, but we repeat it here to get shorter DBUG log file. @@ -64,31 +65,51 @@ int heap_check_heap(HP_INFO *info, my_bo { if (pos < next_block) { - info->current_ptr+= share->block.recbuffer; + info->current_ptr+= share->recordspace.block.recbuffer; } else { - next_block+= share->block.records_in_block; - if (next_block >= share->records+share->deleted) + next_block+= share->recordspace.block.records_in_block; + if (next_block >= share->recordspace.chunk_count) { - next_block= share->records+share->deleted; - if (pos >= next_block) - break; /* End of file */ + next_block= share->recordspace.chunk_count; + if (pos >= next_block) + break; /* End of file */ } } + hp_find_record(info,pos); - if (!info->current_ptr[share->reclength]) - deleted++; - else - records++; + switch (get_chunk_status(&share->recordspace, info->current_ptr)) { + case CHUNK_STATUS_DELETED: + deleted++; + chunk_count++; + break; + case CHUNK_STATUS_ACTIVE: + records++; + chunk_count++; + break; + case CHUNK_STATUS_LINKED: + chunk_count++; + break; + default: + DBUG_PRINT("error", + ("Unknown record status: Record: 0x%lx Status %lu", + (long) info->current_ptr, (ulong) get_chunk_status(&share->recordspace, info->current_ptr))); + error|= 1; + break; + } } - if (records != share->records || deleted != share->deleted) + /* TODO: verify linked chunks (no orphans, no cycles, no bad links) */ + + if (records != share->records || chunk_count != share->recordspace.chunk_count || + deleted != share->recordspace.del_chunk_count) { - DBUG_PRINT("error",("Found rows: %lu (%lu) deleted %lu (%lu)", + DBUG_PRINT("error",("Found rows: %lu (%lu) total chunks %lu (%lu) deleted chunks %lu (%lu)", records, (ulong) share->records, - deleted, (ulong) share->deleted)); + chunk_count, (ulong) share->recordspace.chunk_count, + deleted, (ulong) share->recordspace.del_chunk_count)); error= 1; } *info= save_info; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/_rectest.c ./heap/_rectest.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/_rectest.c 2007-07-04 06:06:58.000000000 -0700 +++ ./heap/_rectest.c 2008-02-25 13:43:16.000000000 -0800 @@ -18,13 +18,14 @@ #include "heapdef.h" -int hp_rectest(register HP_INFO *info, register const byte *old) +int hp_rectest(register HP_INFO *info, register const byte *old_record) { DBUG_ENTER("hp_rectest"); - if (memcmp(info->current_ptr,old,(size_t) info->s->reclength)) + if (hp_process_record_data_to_chunkset(info->s, old_record, info->current_ptr, 1)) { DBUG_RETURN((my_errno=HA_ERR_RECORD_CHANGED)); /* Record have changed */ } + DBUG_RETURN(0); } /* _heap_rectest */ diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/heapdef.h ./heap/heapdef.h --- D:\projects\mysql\mysql-5.0.45-orig/heap/heapdef.h 2007-07-04 06:06:10.000000000 -0700 +++ ./heap/heapdef.h 2008-02-25 13:36:02.000000000 -0800 @@ -33,6 +33,10 @@ #define HP_MIN_RECORDS_IN_BLOCK 16 #define HP_MAX_RECORDS_IN_BLOCK 8192 +#define CHUNK_STATUS_DELETED 0 /* this chunk has been deleted and can be reused */ +#define CHUNK_STATUS_ACTIVE 1 /* this chunk represents the first part of a live record */ +#define CHUNK_STATUS_LINKED 2 /* this chunk is a continuation from another chunk (part of chunkset) */ + /* Some extern variables */ extern LIST *heap_open_list,*heap_share_list; @@ -43,7 +47,11 @@ if (!(info->update & HA_STATE_AKTIV))\ #define hp_find_hash(A,B) ((HASH_INFO*) hp_find_block((A),(B))) /* Find pos for record and update it in info->current_ptr */ -#define hp_find_record(info,pos) (info)->current_ptr= hp_find_block(&(info)->s->block,pos) +#define hp_find_record(info,pos) (info)->current_ptr= hp_find_block(&(info)->s->recordspace.block,pos) + +#define get_chunk_status(info,ptr) (ptr[(info)->offset_status]) + +#define get_chunk_count(info,rec_length) ((rec_length + (info)->chunk_dataspace_length - 1) / (info)->chunk_dataspace_length) typedef struct st_hp_hash_info { @@ -101,6 +109,19 @@ extern void hp_clear(HP_SHARE *info); extern void hp_clear_keys(HP_SHARE *info); extern uint hp_rb_pack_key(HP_KEYDEF *keydef, uchar *key, const uchar *old, uint k_len); + + /* Chunkset management (alloc/free/encode/decode) functions */ + +extern byte *hp_allocate_chunkset(HP_DATASPACE *info, uint chunk_count); +extern int hp_reallocate_chunkset(HP_DATASPACE *info, uint chunk_count, byte* pos); +extern void hp_free_chunks(HP_DATASPACE *info, byte *pos); +extern void hp_clear_dataspace(HP_DATASPACE *info); + +extern uint hp_get_encoded_data_length(HP_SHARE *info, const byte *record, uint *chunk_count); +extern void hp_copy_record_data_to_chunkset(HP_SHARE *info, const byte *record, byte *pos); +extern void hp_extract_record(HP_SHARE *info, byte *record, const byte *pos); +extern uint hp_process_record_data_to_chunkset(HP_SHARE *info, const byte *record, byte *pos, uint is_compare); + #ifdef THREAD extern pthread_mutex_t THR_LOCK_heap; #else diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_clear.c ./heap/hp_clear.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_clear.c 2007-07-04 06:06:16.000000000 -0700 +++ ./heap/hp_clear.c 2008-02-04 21:19:18.000000000 -0800 @@ -30,16 +30,11 @@ void hp_clear(HP_SHARE *info) { DBUG_ENTER("hp_clear"); - if (info->block.levels) - VOID(hp_free_level(&info->block,info->block.levels,info->block.root, - (byte*) 0)); - info->block.levels=0; + hp_clear_dataspace(&info->recordspace); hp_clear_keys(info); - info->records= info->deleted= 0; - info->data_length= 0; + info->records= 0; info->blength=1; info->changed=0; - info->del_link=0; DBUG_VOID_RETURN; } @@ -157,7 +152,7 @@ int heap_enable_indexes(HP_INFO *info) int error= 0; HP_SHARE *share= info->s; - if (share->data_length || share->index_length) + if (share->recordspace.total_data_length || share->index_length) error= HA_ERR_CRASHED; else if (share->currently_disabled_keys) @@ -191,4 +186,3 @@ int heap_indexes_are_disabled(HP_INFO *i return (! share->keys && share->currently_disabled_keys); } - diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_create.c ./heap/hp_create.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_create.c 2007-07-04 06:06:08.000000000 -0700 +++ ./heap/hp_create.c 2008-03-04 13:28:40.000000000 -0800 @@ -14,19 +14,32 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include "heapdef.h" +#include +#include static int keys_compare(heap_rb_param *param, uchar *key1, uchar *key2); -static void init_block(HP_BLOCK *block,uint reclength,ulong min_records, +static void init_block(HP_BLOCK *block,uint chunk_length, ulong min_records, ulong max_records); +#define FIXED_REC_OVERHEAD (sizeof(byte)) +#define VARIABLE_REC_OVERHEAD (sizeof(byte**) + sizeof(byte)) + +/* Minimum size that a chunk can take, 12 bytes on 32bit, 24 bytes on 64bit */ +#define VARIABLE_MIN_CHUNK_SIZE \ + ((sizeof(byte**) + VARIABLE_REC_OVERHEAD + sizeof(byte**) - 1) & ~(sizeof(byte**) - 1)) + int heap_create(const char *name, uint keys, HP_KEYDEF *keydef, - uint reclength, ulong max_records, ulong min_records, + uint columns, HP_COLUMNDEF *columndef, + uint max_key_fieldnr, uint key_part_size, + uint reclength, uint keys_memory_size, + ulong max_records, ulong min_records, HP_CREATE_INFO *create_info) { uint i, j, key_segs, max_length, length; + ulong max_rows_for_stated_memory; HP_SHARE *share; HA_KEYSEG *keyseg; - + DBUG_ENTER("heap_create"); pthread_mutex_lock(&THR_LOCK_heap); @@ -35,18 +48,127 @@ int heap_create(const char *name, uint k hp_free(share); share= NULL; } - + if (!share) { + uint chunk_dataspace_length, chunk_length, is_variable_size; + uint fixed_data_length, fixed_column_count; HP_KEYDEF *keyinfo; + DBUG_PRINT("info",("Initializing new table")); - + + if (create_info->max_chunk_size) + { + uint configured_chunk_size= create_info->max_chunk_size; + + /* User requested variable-size records, let's see if they're possible */ + + if (configured_chunk_size < key_part_size) + { + /* Eventual chunk_size cannot be smaller than key data, + which allows all keys to fit into the first chunk */ + my_error(ER_CANT_USE_OPTION_HERE, MYF(0), "block_size"); + return ER_CANT_USE_OPTION_HERE; + } + + if ((reclength - configured_chunk_size) >= VARIABLE_MIN_CHUNK_SIZE<<1) + { + /* Allow variable size only if we're saving at least two smallest chunks */ + /* There has to be at least one field after indexed fields */ + /* Note that NULL bits are already included in key_part_size */ + is_variable_size= 1; + chunk_dataspace_length= configured_chunk_size; + } + else + { + /* max_chunk_size is near the full reclength, let's use fixed size */ + is_variable_size= 0; + chunk_dataspace_length= reclength; + } + } + else if (create_info->is_dynamic) + { + /* User asked for dynamic records - use 256 as the chunk size */ + if ((key_part_size + VARIABLE_REC_OVERHEAD) > 256) + chunk_dataspace_length= key_part_size; + else + chunk_dataspace_length= 256 - VARIABLE_REC_OVERHEAD; + + is_variable_size= 1; + } + else + { + /* if max_chunk_size is not specified, put the whole record in one chunk */ + is_variable_size= 0; + chunk_dataspace_length= reclength; + } + + if (is_variable_size) + { + /* Check whether we have any variable size records past key data */ + uint has_variable_fields= 0; + + fixed_data_length= key_part_size; + fixed_column_count= max_key_fieldnr; + + for (i= max_key_fieldnr; i < columns; i++) + { + HP_COLUMNDEF* column= columndef + i; + if (column->type == MYSQL_TYPE_VARCHAR || column->length >= 32) + { + /* The field has to be >= 5.0.3 true VARCHAR and have substantial length */ + /* TODO: do we want to calculate minimum length? */ + has_variable_fields= 1; + break; + } + + if (has_variable_fields) + { + break; + } + + if ((column->offset + column->length) <= chunk_dataspace_length) + { + /* Still no variable-size columns, add one fixed-length */ + fixed_column_count= i + 1; + fixed_data_length= column->offset + column->length; + } + } + + if (!has_variable_fields) + { + /* There is no need to use variable-size records without variable-size columns */ + /* Reset sizes if it's not variable size anymore */ + is_variable_size= 0; + chunk_dataspace_length= reclength; + fixed_data_length= reclength; + fixed_column_count= columns; + } + } + else + { + fixed_data_length= reclength; + fixed_column_count= columns; + } + /* - We have to store sometimes byte* del_link in records, - so the record length should be at least sizeof(byte*) + We store byte* del_link inside the data area of deleted records, + so the data length should be at least sizeof(byte*) */ - set_if_bigger(reclength, sizeof (byte*)); - + set_if_bigger(chunk_dataspace_length, sizeof (byte**)); + + if (is_variable_size) + { + chunk_length= chunk_dataspace_length + VARIABLE_REC_OVERHEAD; + } + else + { + chunk_length= chunk_dataspace_length + FIXED_REC_OVERHEAD; + } + + /* Align chunk length to the next pointer */ + chunk_length= (uint) (chunk_length + sizeof(byte**) - 1) & ~(sizeof(byte**) - 1); + for (i= key_segs= max_length= 0, keyinfo= keydef; i < keys; i++, keyinfo++) { bzero((char*) &keyinfo->block,sizeof(keyinfo->block)); @@ -129,16 +251,31 @@ int heap_create(const char *name, uint k } if (!(share= (HP_SHARE*) my_malloc((uint) sizeof(HP_SHARE)+ keys*sizeof(HP_KEYDEF)+ + columns*sizeof(HP_COLUMNDEF)+ key_segs*sizeof(HA_KEYSEG), MYF(MY_ZEROFILL)))) { pthread_mutex_unlock(&THR_LOCK_heap); DBUG_RETURN(1); } - share->keydef= (HP_KEYDEF*) (share + 1); + + /* + Max_records is used for estimating block sizes and for enforcement. + Calculate the very maximum number of rows (if everything was one chunk) and + then take either that value or configured max_records (pick smallest one) + */ + max_rows_for_stated_memory= (ha_rows) (create_info->max_table_size / + (keys_memory_size + chunk_length)); + max_records = ((max_records && max_records < max_rows_for_stated_memory) ? + max_records : max_rows_for_stated_memory); + + share->column_defs= (HP_COLUMNDEF*) (share + 1); + memcpy(share->column_defs, columndef, (size_t) (sizeof(columndef[0]) * columns)); + + share->keydef= (HP_KEYDEF*) (share->column_defs + columns); share->key_stat_version= 1; keyseg= (HA_KEYSEG*) (share->keydef + keys); - init_block(&share->block, reclength + 1, min_records, max_records); + init_block(&share->recordspace.block, chunk_length, min_records, max_records); /* Fix keys */ memcpy(share->keydef, keydef, (size_t) (sizeof(keydef[0]) * keys)); for (i= 0, keyinfo= share->keydef; i < keys; i++, keyinfo++) @@ -176,15 +313,32 @@ int heap_create(const char *name, uint k share->min_records= min_records; share->max_records= max_records; share->max_table_size= create_info->max_table_size; - share->data_length= share->index_length= 0; - share->reclength= reclength; + share->index_length= 0; share->blength= 1; share->keys= keys; share->max_key_length= max_length; + share->column_count= columns; share->changed= 0; share->auto_key= create_info->auto_key; share->auto_key_type= create_info->auto_key_type; share->auto_increment= create_info->auto_increment; + + share->fixed_data_length= fixed_data_length; + share->fixed_column_count= fixed_column_count; + + share->recordspace.chunk_length= chunk_length; + share->recordspace.chunk_dataspace_length= chunk_dataspace_length; + share->recordspace.is_variable_size= is_variable_size; + share->recordspace.total_data_length= 0; + + if (is_variable_size) { + share->recordspace.offset_link= chunk_dataspace_length; + share->recordspace.offset_status= share->recordspace.offset_link + sizeof(byte**); + } else { + share->recordspace.offset_link= 1<<22; /* Make it likely to fail if anyone uses this offset */ + share->recordspace.offset_status= chunk_dataspace_length; + } + /* Must be allocated separately for rename to work */ if (!(share->name= my_strdup(name,MYF(0)))) { @@ -210,15 +364,18 @@ static int keys_compare(heap_rb_param *p param->search_flag, not_used); } -static void init_block(HP_BLOCK *block, uint reclength, ulong min_records, +static void init_block(HP_BLOCK *block, uint chunk_length, ulong min_records, ulong max_records) { - uint i,recbuffer,records_in_block; + uint i,records_in_block,recbuffer; max_records= max(min_records,max_records); if (!max_records) max_records= 1000; /* As good as quess as anything */ - recbuffer= (uint) (reclength + sizeof(byte**) - 1) & ~(sizeof(byte**) - 1); + + /* we want to start each chunk at 8 bytes boundary, round recbuffer to the next 8 */ + recbuffer= (uint) (chunk_length + sizeof(byte**) - 1) & ~(sizeof(byte**) - 1); + records_in_block= max_records / 10; if (records_in_block < 10 && max_records) records_in_block= 10; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_delete.c ./heap/hp_delete.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_delete.c 2007-07-04 06:05:36.000000000 -0700 +++ ./heap/hp_delete.c 2008-02-04 21:16:48.000000000 -0800 @@ -22,6 +22,8 @@ int heap_delete(HP_INFO *info, const byt byte *pos; HP_SHARE *share=info->s; HP_KEYDEF *keydef, *end, *p_lastinx; + uint rec_length, chunk_count; + DBUG_ENTER("heap_delete"); DBUG_PRINT("enter",("info: 0x%lx record: 0x%lx", (long) info, (long) record)); @@ -31,6 +33,8 @@ int heap_delete(HP_INFO *info, const byt DBUG_RETURN(my_errno); /* Record changed */ share->changed=1; + rec_length = hp_get_encoded_data_length(share, record, &chunk_count); + if ( --(share->records) < share->blength >> 1) share->blength>>=1; pos=info->current_ptr; @@ -43,10 +47,7 @@ int heap_delete(HP_INFO *info, const byt } info->update=HA_STATE_DELETED; - *((byte**) pos)=share->del_link; - share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ - share->deleted++; + hp_free_chunks(&share->recordspace, pos); info->current_hash_ptr=0; #if !defined(DBUG_OFF) && defined(EXTRA_HEAP_DEBUG) DBUG_EXECUTE("check_heap",heap_check_heap(info, 0);); @@ -115,6 +116,9 @@ int hp_delete_key(HP_INFO *info, registe blength=share->blength; if (share->records+1 == blength) blength+= blength; + + /* find the very last HASH_INFO pointer in the index */ + /* note that records has already been decremented */ lastpos=hp_find_hash(&keyinfo->block,share->records); last_ptr=0; @@ -146,16 +150,24 @@ int hp_delete_key(HP_INFO *info, registe (long) info->current_ptr)); } empty=pos; - if (gpos) - gpos->next_key=pos->next_key; /* unlink current ptr */ + if (gpos) { + /* gpos says we have previous HASH_INFO, change previous to point to next, this way unlinking "empty" */ + gpos->next_key=pos->next_key; + } else if (pos->next_key) { + /* no previous gpos, this pos is the first in the list and it has pointer to "next" */ + /* move next HASH_INFO data to our pos, to free up space at the next position */ + /* remember next pos as "empty", nobody refers to "empty" at this point */ empty=pos->next_key; pos->ptr_to_rec=empty->ptr_to_rec; pos->next_key=empty->next_key; } else + { + /* this was the only HASH_INFO at this position */ keyinfo->hash_buckets--; + } if (empty == lastpos) /* deleted last hash key */ DBUG_RETURN (0); diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_dspace.c ./heap/hp_dspace.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_dspace.c 1969-12-31 16:00:00.000000000 -0800 +++ ./heap/hp_dspace.c 2008-04-16 22:35:32.500000000 -0700 @@ -0,0 +1,424 @@ +/* Copyright (C) 2000-2002 MySQL AB + Copyright (C) 2008 eBay, Inc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Implements various base dataspace-related functions - allocate, free, clear */ + +#include "heapdef.h" + + +/* + MySQL Heap tables keep data in arrays of fixed-size chunks. + These chunks are organized into two groups of HP_BLOCK structures: + - group1 contains indexes, with one HP_BLOCK per key + (part of HP_KEYDEF) + - group2 contains record data, with single HP_BLOCK + for all records, referenced by HP_SHARE.recordspace.block + + While columns used in index are usually small, other columns + in the table may need to accomodate larger data. Typically, + larger data is placed into VARCHAR or BLOB columns. With actual + sizes varying, Heap Engine has to support variable-sized records + in memory. Heap Engine implements the concept of dataspace + (HP_DATASPACE), which incorporates HP_BLOCK for the record data, + and adds more information for managing variable-sized records. + + Variable-size records are stored in multiple "chunks", + which means that a single record of data (database "row") can + consist of multiple chunks organized into one "set". HP_BLOCK + contains chunks. In variable-size format, one record + is represented as one or many chunks, depending on the actual + data, while in fixed-size mode, one record is always represented + as one chunk. The index structures would always point to the first + chunk in the chunkset. + + At the time of table creation, Heap Engine attempts to find out + if variable-size records are desired. A user can request + variable-size records by providing either row_type=dynamic or + block_size=NNN table create option. Heap Engine will check + whether block_size provides enough space in the first chunk + to keep all null bits and columns that are used in indexes. + If block_size is too small, table creation will be aborted + with an error. Heap Engine will revert to fixed-size allocation + mode if block_size provides no memory benefits (no VARCHAR + fields extending past first chunk). + + In order to improve index search performance, Heap Engine needs + to keep all null flags and all columns used as keys inside + the first chunk of a chunkset. In particular, this means that + all columns used as keys should be defined first in the table + creation SQL. The length of data used by null bits and key columns + is stored as fixed_data_length inside HP_SHARE. fixed_data_length + will extend past last key column if more fixed-length fields can + fit into the first chunk. + + Variable-size records are necessary only in the presence + of variable-size columns. Heap Engine will be looking for VARCHAR + columns, which declare length of 32 or more. If no such columns + are found, table will be switched to fixed-size format. You should + always try to put such columns at the end of the table definition. + + Whenever data is being inserted or updated in the table + Heap Engine will calculate how many chunks are necessary. + For insert operations, Heap Engine allocates new chunkset in + the recordspace. For update operations it will modify length of + the existing chunkset, unlinking unnecessary chunks at the end, + or allocating and adding more if larger length is necessary. + + When writing data to chunks or copying data back to record, + Heap Engine will first copy fixed_data_length of data using single + memcpy call. The rest of the columns are processed one-by-one. + Non-VARCHAR columns are copied in their full format. VARCHAR's + are copied based on their actual length. Any NULL values after + fixed_data_length are skipped. + + The allocation and contents of the actual chunks varies between + fixed and variable-size modes. Total chunk length is always + aligned to the next sizeof(byte*). Here is the format of + fixed-size chunk: + byte[] - sizeof=chunk_dataspace_length, but at least + sizeof(byte*) bytes. Keeps actual data or pointer + to the next deleted chunk. + chunk_dataspace_length equals to full record length + byte - status field (1 means "in use", 0 means "deleted") + Variable-size uses different format: + byte[] - sizeof=chunk_dataspace_length, but at least + sizeof(byte*) bytes. Keeps actual data or pointer + to the next deleted chunk. + chunk_dataspace_length is set according to table + setup (block_size) + byte* - pointer to the next chunk in this chunkset, + or NULL for the last chunk + byte - status field (1 means "first", 0 means "deleted", + 2 means "linked") + + When allocating a new chunkset of N chunks, Heap Engine will try + to allocate chunks one-by-one, linking them as they become + allocated. Allocation of a single chunk will attempt to reuse + a deleted (freed) chunk. If no free chunks are available, + it will attempt to allocate a new area inside HP_BLOCK. + Freeing chunks will place them at the front of free list + referenced by del_link in HP_DATASPACE. The newly freed chunk + will contain reference to the previously freed chunk in its first + sizeof(byte*) of the payload space. + + Here is open issues: + 1. It is not very nice to require people to keep key columns + at the beginning of the table creation SQL. There are three + proposed resolutions: + a. Leave it as is. It's a reasonable limitation + b. Add new HA_KEEP_KEY_COLUMNS_TO_FRONT flag to handler.h and + make table.cpp align columns when it creates the table + c. Make HeapEngine reorder columns in the chunk data, so that + key columns go first. Add parallel HA_KEYSEG structures + to distinguish positions in record vs. positions in + the first chunk. Copy all data field-by-field rather than + using single memcpy unless DBA kept key columns to + the beginning. + 2. heap_check_heap needs verify linked chunks, looking for + issues such as orphans, cycles, and bad links. However, + Heap Engine today does not do similar things even for + free list. + 3. With new HP_DATASPACE allocation mechaism, BLOB will become + increasingly simple to implement, but I may not have time + for that. In one approach, BLOB data can be placed at + the end of the same record. In another approach (which I + prefer) BLOB data would have its own HP_DATASPACE with + variable-size entries. + 4. In a more sophisticated implementation, some space can + be saved even with all fixed-size columns if many of them + have NULL value, as long as these columns are not used + in indexes + 5. In variable-size format status should be moved to lower + bits of the "next" pointer. Pointer is always aligned + to sizeof(byte*), which is at least 4, leaving 2 lower + bits free. This will save 8 bytes per chunk + on 64-bit platform. + 6. As we do not want to modify FRM format, BLOCK_SIZE option + of "CREATE TABLE" is saved as "RAID_CHUNKSIZE" for + Heap Engine tables. +*/ + +static byte *hp_allocate_one_chunk(HP_DATASPACE *info); + + +/** + Clear a dataspace + + Frees memory and zeros-out any relevant counters in the dataspace + + @param info the dataspace to clear +*/ + +void hp_clear_dataspace(HP_DATASPACE *info) +{ + if (info->block.levels) + { + VOID(hp_free_level(&info->block,info->block.levels,info->block.root, + (byte*) 0)); + } + info->block.levels=0; + info->del_chunk_count= info->chunk_count= 0; + info->del_link=0; + info->total_data_length= 0; +} + + +/** + Allocate or reallocate a chunkset in the dataspace + + Attempts to allocate a new chunkset or change the size of an existing chunkset + + @param info the hosting dataspace + @param chunk_count the number of chunks that we expect as the result + @param existing_set non-null value asks function to resize existing chunkset, + return value would point to this set + + @return Pointer to the first chunk in the new or updated chunkset, or NULL if unsuccessful +*/ + +static byte *hp_allocate_variable_chunkset(HP_DATASPACE *info, + uint chunk_count, byte* existing_set) +{ + int alloc_count= chunk_count, i; + byte *first_chunk= 0, *curr_chunk= 0, *prev_chunk= 0, *last_existing_chunk= 0; + + DBUG_ASSERT(alloc_count); + + if (existing_set) + { + first_chunk= existing_set; + + curr_chunk= existing_set; + while (curr_chunk && alloc_count) + { + prev_chunk= curr_chunk; + curr_chunk= *((byte**)(curr_chunk + info->offset_link)); + alloc_count--; + } + + if (!alloc_count) + { + if (curr_chunk) + { + /* We came through all chunks and there is more left, let's truncate the list */ + *((byte**)(prev_chunk + info->offset_link)) = NULL; + hp_free_chunks(info, curr_chunk); + } + + return first_chunk; + } + + last_existing_chunk = prev_chunk; + } + + /* We can reach this point only if we're allocating new chunkset or more chunks in existing set */ + + for (i=0; ioffset_link)); + *((byte**)(prev_chunk + info->offset_link)) = NULL; + hp_free_chunks(info, curr_chunk); + } + else if (first_chunk) + { + /* free any chunks previously allocated */ + hp_free_chunks(info, first_chunk); + } + + return NULL; + } + + /* mark as if this chunk is last in the chunkset */ + *((byte**) (curr_chunk + info->offset_link))= 0; + + if (prev_chunk) + { + /* tie them into a linked list */ + *((byte**) (prev_chunk + info->offset_link))= curr_chunk; + curr_chunk[info->offset_status]= CHUNK_STATUS_LINKED; /* Record linked from active */ + } + else + { + curr_chunk[info->offset_status]= CHUNK_STATUS_ACTIVE; /* Record active */ + } + + if (!first_chunk) + { + first_chunk= curr_chunk; + } + + prev_chunk= curr_chunk; + } + + return first_chunk; +} + + +/** + Allocate a new chunkset in the dataspace + + Attempts to allocate a new chunkset + + @param info the hosting dataspace + @param chunk_count the number of chunks that we expect as the result + + @return Pointer to the first chunk in the new or updated chunkset, or NULL if unsuccessful +*/ + +byte *hp_allocate_chunkset(HP_DATASPACE *info, uint chunk_count) +{ + byte* result; + + DBUG_ENTER("hp_allocate_chunks"); + + if (info->is_variable_size) + { + result = hp_allocate_variable_chunkset(info, chunk_count, NULL); + } + else + { + result= hp_allocate_one_chunk(info); + if (result) + { + result[info->offset_status]= CHUNK_STATUS_ACTIVE; + } + + DBUG_RETURN(result); + } + + DBUG_RETURN(result); +} + + +/** + Reallocate an existing chunkset in the dataspace + + Attempts to change the size of an existing chunkset + + @param info the hosting dataspace + @param chunk_count the number of chunks that we expect as the result + @param pos pointer to the existing chunkset + + @return Error code or zero if successful +*/ + +int hp_reallocate_chunkset(HP_DATASPACE *info, uint chunk_count, byte* pos) +{ + DBUG_ENTER("hp_reallocate_chunks"); + + if (!info->is_variable_size) + { + /* Update should never change chunk_count in fixed-size mode */ + my_errno=HA_ERR_WRONG_COMMAND; + return my_errno; + } + + /* Reallocate never moves the first chunk */ + if (!hp_allocate_variable_chunkset(info, chunk_count, pos)) + DBUG_RETURN(my_errno); + + DBUG_RETURN(0); +} + + +/** + Allocate a single chunk in the dataspace + + Attempts to allocate a new chunk or reuse one from deleted list + + @param info the hosting dataspace + + @return Pointer to the chunk, or NULL if unsuccessful +*/ + +static byte *hp_allocate_one_chunk(HP_DATASPACE *info) +{ + byte* curr_chunk; + ulong length, block_pos; + + if (info->del_link) + { + curr_chunk=info->del_link; + info->del_link= *((byte**) curr_chunk); + info->del_chunk_count--; + + DBUG_PRINT("hp_allocate_one_chunk",("Used old position: 0x%lx",(long) curr_chunk)); + return curr_chunk; + } + + block_pos= (info->chunk_count % info->block.records_in_block); + if (!block_pos) + { + if (hp_get_new_block(&info->block,&length)) + { + /* no space in the current block */ + return NULL; + } + + info->total_data_length+= length; + } + + info->chunk_count++; + curr_chunk= ((byte*) info->block.level_info[0].last_blocks + + block_pos * info->block.recbuffer); + + DBUG_PRINT("hp_allocate_one_chunk",("Used new position: 0x%lx", (long) curr_chunk)); + + return curr_chunk; +} + + +/** + Free a list of chunks + + Reclaims all chunks linked by the pointer, + which could be the whole chunkset or a part of an existing chunkset + + @param info the hosting dataspace + @param pos pointer to the head of the chunkset +*/ + +void hp_free_chunks(HP_DATASPACE *info, byte *pos) +{ + byte* curr_chunk= pos; + + while (curr_chunk) { + info->del_chunk_count++; + *((byte**) curr_chunk)= info->del_link; + info->del_link= curr_chunk; + + curr_chunk[info->offset_status]= CHUNK_STATUS_DELETED; + + DBUG_PRINT("hp_free_chunks",("Freed position: 0x%lx", (long) curr_chunk)); + + if (!info->is_variable_size) + { + break; + } + + /* Delete next chunk in this chunkset */ + curr_chunk= *((byte**)(curr_chunk + info->offset_link)); + } +} diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_info.c ./heap/hp_info.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_info.c 2007-07-04 06:06:38.000000000 -0700 +++ ./heap/hp_info.c 2008-02-28 08:43:18.000000000 -0800 @@ -47,9 +47,21 @@ int heap_info(reg1 HP_INFO *info,reg2 HE { DBUG_ENTER("heap_info"); x->records = info->s->records; - x->deleted = info->s->deleted; - x->reclength = info->s->reclength; - x->data_length = info->s->data_length; + x->deleted = info->s->recordspace.del_chunk_count; + + if (info->s->recordspace.is_variable_size) + { + if (info->s->records) + x->reclength = (uint)(info->s->recordspace.total_data_length / (ulonglong)info->s->records); + else + x->reclength = info->s->recordspace.chunk_length; + } + else + { + x->reclength = info->s->recordspace.chunk_dataspace_length; + } + + x->data_length = info->s->recordspace.total_data_length; x->index_length = info->s->index_length; x->max_records = info->s->max_records; x->errkey = info->errkey; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_open.c ./heap/hp_open.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_open.c 2007-07-04 06:05:46.000000000 -0700 +++ ./heap/hp_open.c 2008-02-04 16:52:34.000000000 -0800 @@ -62,9 +62,9 @@ HP_INFO *heap_open(const char *name, int #ifndef DBUG_OFF info->opt_flag= READ_CHECK_USED; /* Check when changing */ #endif - DBUG_PRINT("exit",("heap: 0x%lx reclength: %d records_in_block: %d", - (long) info, share->reclength, - share->block.records_in_block)); + DBUG_PRINT("exit",("heap: 0x%lx chunk_length: %d records_in_block: %d", + (long) info, share->recordspace.chunk_length, + share->recordspace.block.records_in_block)); DBUG_RETURN(info); } diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_record.c ./heap/hp_record.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_record.c 1969-12-31 16:00:00.000000000 -0800 +++ ./heap/hp_record.c 2008-04-16 22:12:09.234375000 -0700 @@ -0,0 +1,371 @@ +/* Copyright (C) 2000-2002 MySQL AB + Copyright (C) 2008 eBay, Inc + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ + +/* Implements various base record-related functions, such as encode and decode into chunks */ + +#include "heapdef.h" +#include + + +/** + Calculate size of the record for the purpose of storing in chunks + + Walk through the fields of the record and calculates the exact space + needed in chunks as well the the total chunk count + + @param info the hosting table + @param record the record in standard unpacked format + @param[out] chunk_count the number of chunks needed for this record + + @return The size of the required storage in bytes +*/ + +uint hp_get_encoded_data_length(HP_SHARE *info, const byte *record, uint *chunk_count) +{ + uint i, dst_offset; + + dst_offset= info->fixed_data_length; + + if (!info->recordspace.is_variable_size) + { + /* Nothing more to copy */ + *chunk_count= 1; + return dst_offset; + } + + for (i= info->fixed_column_count; i < info->column_count; i++) + { + uint src_offset, length; + + HP_COLUMNDEF* column= info->column_defs + i; + + if (column->null_bit) + { + if (record[column->null_pos] & column->null_bit) + { + /* Skip all NULL values */ + continue; + } + } + + src_offset= column->offset; + if (column->type == MYSQL_TYPE_VARCHAR) + { + uint pack_length; + + /* >= 5.0.3 true VARCHAR */ + + pack_length= column->length_bytes; + length= pack_length + (pack_length == 1 ? + (uint) *(uchar*) (record + src_offset) : uint2korr(record + src_offset)); + } + else + { + length= column->length; + } + + dst_offset+= length; + } + + *chunk_count= get_chunk_count(&info->recordspace, dst_offset); + + return dst_offset; +} + + +/*static void dump_chunk(HP_SHARE *info, const byte* curr_chunk) +{ + uint i; + fprintf(stdout, "Chunk dump at 0x%lx: ", (long)curr_chunk); + for (i= 0; i < info->chunk_dataspace_length; i++) + { + uint b= *((uchar*)(curr_chunk + i)); + if (b < 0x10) + { + fprintf(stdout, "0"); + } + fprintf(stdout, "%lx ", (long)b); + } + fprintf(stdout, ". Next = 0x%lx, Status = %d\n", + (long)(*((byte**) (curr_chunk + info->offset_link))), + (uint)(*((uchar*) (curr_chunk + info->offset_status)))); +}*/ + + +/** + Encodes or compares record + + Copies data from original unpacked record into the preallocated chunkset, + or performs data comparison + + @param info the hosting table + @param record the record in standard unpacked format + @param pos the target chunkset + @param is_compare flag indicating whether we should compare data or store it + + @return Status of comparison + @retval non-zero if comparison fond data differences + @retval zero otherwise +*/ + +uint hp_process_record_data_to_chunkset(HP_SHARE *info, const byte *record, + byte *pos, uint is_compare) +{ + uint i, dst_offset; + byte* curr_chunk= pos; + + if (is_compare) + { + if (memcmp(curr_chunk, record, (size_t) info->fixed_data_length)) + { + return 1; + } + } + else + { + memcpy(curr_chunk, record, (size_t) info->fixed_data_length); + } + + if (!info->recordspace.is_variable_size) + { + /* Nothing more to copy */ + return 0; + } + + dst_offset= info->fixed_data_length; + + for (i= info->fixed_column_count; i < info->column_count; i++) + { + uint src_offset, length; + + HP_COLUMNDEF* column= info->column_defs + i; + + if (column->null_bit) + { + if (record[column->null_pos] & column->null_bit) + { + /* Skip all NULL values */ + continue; + } + } + + src_offset= column->offset; + if (column->type == MYSQL_TYPE_VARCHAR) + { + uint pack_length; + + /* >= 5.0.3 true VARCHAR */ + + /* Make sure to copy length indicator and actuals string bytes */ + pack_length= column->length_bytes; + length= pack_length + (pack_length == 1 ? + (uint) *(uchar*) (record + src_offset) : uint2korr(record + src_offset)); + } + else + { + length= column->length; + } + + while (length > 0) + { + uint to_copy; + + to_copy= info->recordspace.chunk_dataspace_length - dst_offset; + if (to_copy == 0) + { + /* Jump to the next chunk */ + /*dump_chunk(info, curr_chunk);*/ + curr_chunk= *((byte**) (curr_chunk + info->recordspace.offset_link)); + dst_offset= 0; + continue; + } + + to_copy= min(length, to_copy); + + if (is_compare) + { + if (memcmp(curr_chunk + dst_offset, record + src_offset, (size_t) to_copy)) + { + return 1; + } + } + else + { + memcpy(curr_chunk + dst_offset, record + src_offset, (size_t) to_copy); + } + + src_offset+= to_copy; + dst_offset+= to_copy; + length-= to_copy; + } + } + + /*dump_chunk(info, curr_chunk);*/ + return 0; +} + + +/** + Stores record in the heap table chunks + + Copies data from original unpacked record into the preallocated chunkset + + @param info the hosting table + @param record the record in standard unpacked format + @param pos the target chunkset +*/ + +void hp_copy_record_data_to_chunkset(HP_SHARE *info, const byte *record, byte *pos) +{ + DBUG_ENTER("hp_copy_record_data_to_chunks"); + + hp_process_record_data_to_chunkset(info, record, pos, 0); + + DBUG_VOID_RETURN; +} + + +/* + Macro to switch curr_chunk to the next chunk in the chunkset and reset src_offset +*/ +#define SWITCH_TO_NEXT_CHUNK_FOR_READ(info, curr_chunk, src_offset) \ + { \ + curr_chunk= *((byte**) (curr_chunk + info->recordspace.offset_link)); \ + src_offset= 0; \ + /*dump_chunk(info, curr_chunk);*/ \ + } + + +/** + Copies record data from storage to unpacked record format + + Copies data from chunkset into its original unpacked record + + @param info the hosting table + @param[out] record the target record in standard unpacked format + @param pos the source chunkset +*/ + +void hp_extract_record(HP_SHARE *info, byte *record, const byte *pos) +{ + uint i, src_offset; + const byte* curr_chunk= pos; + + DBUG_ENTER("hp_extract_record"); + + /*if (info->is_variable_size) + { + dump_chunk(info, curr_chunk); + }*/ + + memcpy(record, curr_chunk, (size_t) info->fixed_data_length); + + if (!info->recordspace.is_variable_size) + { + /* Nothing more to copy */ + DBUG_VOID_RETURN; + } + + src_offset= info->fixed_data_length; + + for (i= info->fixed_column_count; i < info->column_count; i++) + { + uint dst_offset, length, is_null = 0; + + HP_COLUMNDEF* column= info->column_defs + i; + + if (column->null_bit) + { + if (record[column->null_pos] & column->null_bit) + { + is_null = 1; + } + } + + dst_offset= column->offset; + if (column->type == MYSQL_TYPE_VARCHAR) + { + uint pack_length, byte1, byte2; + + /* >= 5.0.3 true VARCHAR */ + + if (is_null) + { + /* TODO: is memset really needed? */ + memset(record + column->offset, 0, column->length); + continue; + } + + pack_length= column->length_bytes; + + if (src_offset == info->recordspace.chunk_dataspace_length) + { + SWITCH_TO_NEXT_CHUNK_FOR_READ(info, curr_chunk, src_offset); + } + byte1= *(uchar*) (curr_chunk + src_offset++); + *(record + dst_offset++)= byte1; + + if (pack_length == 1) + { + length= byte1; + } + else + { + if (src_offset == info->recordspace.chunk_dataspace_length) + { + SWITCH_TO_NEXT_CHUNK_FOR_READ(info, curr_chunk, src_offset); + } + byte2= *(uchar*) (curr_chunk + src_offset++); + *(record + dst_offset++)= byte2; + + /* We copy byte-by-byte and then use uint2korr to combine bytes in the right order */ + length= uint2korr(record + dst_offset - 2); + } + } + else + { + if (is_null) + { + /* TODO: is memset really needed? */ + memset(record + column->offset, 0, column->length); + continue; + } + + length= column->length; + } + + while (length > 0) + { + uint to_copy; + + to_copy= info->recordspace.chunk_dataspace_length - src_offset; + if (to_copy == 0) + { + SWITCH_TO_NEXT_CHUNK_FOR_READ(info, curr_chunk, src_offset); + to_copy= info->recordspace.chunk_dataspace_length; + } + + to_copy= min(length, to_copy); + + memcpy(record + dst_offset, curr_chunk + src_offset, (size_t) to_copy); + src_offset+= to_copy; + dst_offset+= to_copy; + length-= to_copy; + } + } + + DBUG_VOID_RETURN; +} diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rfirst.c ./heap/hp_rfirst.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rfirst.c 2007-07-04 06:06:38.000000000 -0700 +++ ./heap/hp_rfirst.c 2008-02-04 21:16:02.000000000 -0800 @@ -34,7 +34,7 @@ int heap_rfirst(HP_INFO *info, byte *rec memcpy(&pos, pos + (*keyinfo->get_key_length)(keyinfo, pos), sizeof(byte*)); info->current_ptr = pos; - memcpy(record, pos, (size_t)share->reclength); + hp_extract_record(share, record, pos); info->update = HA_STATE_AKTIV; } else diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rkey.c ./heap/hp_rkey.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rkey.c 2007-07-04 06:06:50.000000000 -0700 +++ ./heap/hp_rkey.c 2008-01-29 16:44:26.000000000 -0800 @@ -66,7 +66,7 @@ int heap_rkey(HP_INFO *info, byte *recor if (!(keyinfo->flag & HA_NOSAME) || (keyinfo->flag & HA_END_SPACE_KEY)) memcpy(info->lastkey, key, (size_t) keyinfo->length); } - memcpy(record, pos, (size_t) share->reclength); + hp_extract_record(share, record, pos); info->update= HA_STATE_AKTIV; DBUG_RETURN(0); } diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rlast.c ./heap/hp_rlast.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rlast.c 2007-07-04 06:06:40.000000000 -0700 +++ ./heap/hp_rlast.c 2008-01-29 16:44:20.000000000 -0800 @@ -35,7 +35,7 @@ int heap_rlast(HP_INFO *info, byte *reco memcpy(&pos, pos + (*keyinfo->get_key_length)(keyinfo, pos), sizeof(byte*)); info->current_ptr = pos; - memcpy(record, pos, (size_t)share->reclength); + hp_extract_record(share, record, pos); info->update = HA_STATE_AKTIV; } else diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rnext.c ./heap/hp_rnext.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rnext.c 2007-07-04 06:06:50.000000000 -0700 +++ ./heap/hp_rnext.c 2008-01-29 16:44:14.000000000 -0800 @@ -80,7 +80,7 @@ int heap_rnext(HP_INFO *info, byte *reco my_errno=HA_ERR_END_OF_FILE; DBUG_RETURN(my_errno); } - memcpy(record,pos,(size_t) share->reclength); + hp_extract_record(share, record, pos); info->update=HA_STATE_AKTIV | HA_STATE_NEXT_FOUND; DBUG_RETURN(0); } diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rprev.c ./heap/hp_rprev.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rprev.c 2007-07-04 06:06:26.000000000 -0700 +++ ./heap/hp_rprev.c 2008-01-29 16:44:10.000000000 -0800 @@ -77,7 +77,7 @@ int heap_rprev(HP_INFO *info, byte *reco my_errno=HA_ERR_END_OF_FILE; DBUG_RETURN(my_errno); } - memcpy(record,pos,(size_t) share->reclength); + hp_extract_record(share, record, pos); info->update=HA_STATE_AKTIV | HA_STATE_PREV_FOUND; DBUG_RETURN(0); } diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rrnd.c ./heap/hp_rrnd.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rrnd.c 2007-07-04 06:05:42.000000000 -0700 +++ ./heap/hp_rrnd.c 2008-02-25 13:40:34.000000000 -0800 @@ -36,13 +36,14 @@ int heap_rrnd(register HP_INFO *info, by info->update= 0; DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); } - if (!info->current_ptr[share->reclength]) + if (get_chunk_status(&share->recordspace, info->current_ptr) != CHUNK_STATUS_ACTIVE) { + /* treat deleted and linked chunks as deleted */ info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); } info->update=HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND | HA_STATE_AKTIV; - memcpy(record,info->current_ptr,(size_t) share->reclength); + hp_extract_record(share, record, info->current_ptr); DBUG_PRINT("exit", ("found record at 0x%lx", (long) info->current_ptr)); info->current_hash_ptr=0; /* Can't use rnext */ DBUG_RETURN(0); @@ -70,17 +71,17 @@ int heap_rrnd_old(register HP_INFO *info { pos= ++info->current_record; if (pos % share->block.records_in_block && /* Quick next record */ - pos < share->records+share->deleted && - (info->update & HA_STATE_PREV_FOUND)) + pos < share->used_chunk_count+share->deleted_chunk_count && + (info->update & HA_STATE_PREV_FOUND)) { - info->current_ptr+=share->block.recbuffer; + info->current_ptr+=share->block.recbufferlen; goto end; } } else info->current_record=pos; - if (pos >= share->records+share->deleted) + if (pos >= share->used_chunk_count+share->deleted_chunk_count) { info->update= 0; DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); @@ -90,13 +91,14 @@ int heap_rrnd_old(register HP_INFO *info hp_find_record(info, pos); end: - if (!info->current_ptr[share->reclength]) + if (GET_CHUNK_STATUS(info, info->current_ptr) != CHUNK_STATUS_ACTIVE) { + /* treat deleted and linked chunks as deleted */ info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); } info->update=HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND | HA_STATE_AKTIV; - memcpy(record,info->current_ptr,(size_t) share->reclength); + hp_extract_record(share, record, info->current_ptr); DBUG_PRINT("exit",("found record at 0x%lx",info->current_ptr)); info->current_hash_ptr=0; /* Can't use rnext */ DBUG_RETURN(0); diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rsame.c ./heap/hp_rsame.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_rsame.c 2007-07-04 06:06:50.000000000 -0700 +++ ./heap/hp_rsame.c 2008-02-25 13:41:24.000000000 -0800 @@ -31,7 +31,7 @@ int heap_rsame(register HP_INFO *info, b DBUG_ENTER("heap_rsame"); test_active(info); - if (info->current_ptr[share->reclength]) + if (get_chunk_status(&share->recordspace, info->current_ptr) == CHUNK_STATUS_ACTIVE) { if (inx < -1 || inx >= (int) share->keys) { @@ -47,9 +47,12 @@ int heap_rsame(register HP_INFO *info, b DBUG_RETURN(my_errno); } } - memcpy(record,info->current_ptr,(size_t) share->reclength); + hp_extract_record(share, record, info->current_ptr); DBUG_RETURN(0); } + + /* treat deleted and linked chunks as deleted */ + info->update=0; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_scan.c ./heap/hp_scan.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_scan.c 2007-07-04 06:06:26.000000000 -0700 +++ ./heap/hp_scan.c 2008-02-28 21:06:42.000000000 -0800 @@ -43,30 +43,30 @@ int heap_scan(register HP_INFO *info, by pos= ++info->current_record; if (pos < info->next_block) { - info->current_ptr+=share->block.recbuffer; + info->current_ptr+=share->recordspace.block.recbuffer; } else { - info->next_block+=share->block.records_in_block; - if (info->next_block >= share->records+share->deleted) + info->next_block+=share->recordspace.block.records_in_block; + if (info->next_block >= share->recordspace.chunk_count) { - info->next_block= share->records+share->deleted; + info->next_block= share->recordspace.chunk_count; if (pos >= info->next_block) { - info->update= 0; - DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); + info->update= 0; + DBUG_RETURN(my_errno= HA_ERR_END_OF_FILE); } } hp_find_record(info, pos); } - if (!info->current_ptr[share->reclength]) + if (get_chunk_status(&share->recordspace, info->current_ptr) != CHUNK_STATUS_ACTIVE) { - DBUG_PRINT("warning",("Found deleted record")); + DBUG_PRINT("warning",("Found deleted record or secondary chunk")); info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND; DBUG_RETURN(my_errno=HA_ERR_RECORD_DELETED); } info->update= HA_STATE_PREV_FOUND | HA_STATE_NEXT_FOUND | HA_STATE_AKTIV; - memcpy(record,info->current_ptr,(size_t) share->reclength); + hp_extract_record(share, record, info->current_ptr); info->current_hash_ptr=0; /* Can't use read_next */ DBUG_RETURN(0); } /* heap_scan */ diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_test1.c ./heap/hp_test1.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_test1.c 2007-07-04 06:06:22.000000000 -0700 +++ ./heap/hp_test1.c 2008-02-15 12:11:26.000000000 -0800 @@ -22,6 +22,7 @@ #include #include #include +#include #include "heap.h" static int get_options(int argc, char *argv[]); @@ -35,6 +36,7 @@ int main(int argc, char **argv) char record[128],key[32]; const char *filename; HP_KEYDEF keyinfo[10]; + HP_COLUMNDEF columndef[2]; HA_KEYSEG keyseg[4]; HP_CREATE_INFO hp_create_info; MY_INIT(argv[0]); @@ -53,12 +55,24 @@ int main(int argc, char **argv) keyinfo[0].seg[0].length=6; keyinfo[0].seg[0].charset= &my_charset_latin1; keyinfo[0].flag = HA_NOSAME; - + + memset(columndef, 0, 2 * sizeof(HP_COLUMNDEF)); + columndef[0].type= MYSQL_TYPE_STRING; + columndef[0].offset= 1; + columndef[0].length= 6; + columndef[1].type= MYSQL_TYPE_STRING; + columndef[1].offset= 7; + columndef[1].length= 23; + deleted=0; bzero((gptr) flags,sizeof(flags)); printf("- Creating heap-file\n"); - if (heap_create(filename,1,keyinfo,30,(ulong) flag*100000L,10L, + if (heap_create(filename,1,keyinfo, + 2,columndef, + 1,7, + 30,sizeof(char*) * 2, + (ulong) flag*100000L,10L, &hp_create_info) || !(file= heap_open(filename, 2))) goto err; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_test2.c ./heap/hp_test2.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_test2.c 2007-07-04 06:05:40.000000000 -0700 +++ ./heap/hp_test2.c 2008-02-15 12:17:16.000000000 -0800 @@ -27,6 +27,7 @@ #include "heapdef.h" /* Because of hp_find_block */ #include +#include #define MAX_RECORDS 100000 #define MAX_KEYS 4 @@ -53,6 +54,7 @@ int main(int argc, char *argv[]) register uint i,j; uint ant,n1,n2,n3; uint write_count,update,opt_delete,check2,dupp_keys,found_key; + uint mem_per_keys; int error; ulong pos; unsigned long key_check; @@ -61,6 +63,7 @@ int main(int argc, char *argv[]) HP_INFO *file,*file2; HP_KEYDEF keyinfo[MAX_KEYS]; HA_KEYSEG keyseg[MAX_KEYS*5]; + HP_COLUMNDEF columndef[4]; HEAP_PTR position; HP_CREATE_INFO hp_create_info; CHARSET_INFO *cs= &my_charset_latin1; @@ -121,11 +124,33 @@ int main(int argc, char *argv[]) keyinfo[3].seg[0].null_pos=38; keyinfo[3].seg[0].charset=cs; + memset(columndef, 0, 4 * sizeof(HP_COLUMNDEF)); + columndef[0].type= MYSQL_TYPE_STRING; + columndef[0].offset= 0; + columndef[0].length= 6; + columndef[1].type= MYSQL_TYPE_STRING; + columndef[1].offset= 7; + columndef[1].length= 6; + columndef[2].type= MYSQL_TYPE_STRING; + columndef[2].offset= 12; + columndef[2].length= 8; + columndef[3].type= MYSQL_TYPE_TINY; + columndef[3].offset= 37; + columndef[3].length= 1; + columndef[3].null_bit= 1; + columndef[3].null_pos= 38; + + mem_per_keys= (sizeof(char*) * 2) * 4; + bzero((char*) key1,sizeof(key1)); bzero((char*) key3,sizeof(key3)); printf("- Creating heap-file\n"); - if (heap_create(filename,keys,keyinfo,reclength,(ulong) flag*100000L, + if (heap_create(filename,keys,keyinfo, + 4,columndef, + 4,39, + reclength,mem_per_keys, + (ulong) flag*100000L, (ulong) recant/2, &hp_create_info) || !(file= heap_open(filename, 2))) goto err; @@ -562,7 +587,7 @@ int main(int argc, char *argv[]) heap_close(file2); printf("- Creating output heap-file 2\n"); - if (heap_create(filename2,1,keyinfo,reclength,0L,0L,&hp_create_info) || + if (heap_create(filename2,1,keyinfo,4,columndef,4,39,reclength,mem_per_keys,0L,0L,&hp_create_info) || !(file2= heap_open(filename2, 2))) goto err; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_update.c ./heap/hp_update.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_update.c 2007-07-04 06:06:08.000000000 -0700 +++ ./heap/hp_update.c 2008-02-29 16:13:02.000000000 -0800 @@ -17,43 +17,62 @@ #include "heapdef.h" -int heap_update(HP_INFO *info, const byte *old, const byte *heap_new) +int heap_update(HP_INFO *info, const byte *old_record, const byte *new_record) { HP_KEYDEF *keydef, *end, *p_lastinx; byte *pos; bool auto_key_changed= 0; HP_SHARE *share= info->s; + uint old_length, new_length; + uint old_chunk_count, new_chunk_count; + DBUG_ENTER("heap_update"); test_active(info); pos=info->current_ptr; - if (info->opt_flag & READ_CHECK_USED && hp_rectest(info,old)) + if (info->opt_flag & READ_CHECK_USED && hp_rectest(info,old_record)) DBUG_RETURN(my_errno); /* Record changed */ + + old_length = hp_get_encoded_data_length(share, old_record, &old_chunk_count); + new_length = hp_get_encoded_data_length(share, new_record, &new_chunk_count); + + if (new_chunk_count > old_chunk_count) { + /* extend the old chunkset size as necessary, but do not shrink yet */ + if (hp_reallocate_chunkset(&share->recordspace, new_chunk_count, pos)) { + DBUG_RETURN(my_errno); /* Out of memory or table space */ + } + } + if (--(share->records) < share->blength >> 1) share->blength>>= 1; share->changed=1; p_lastinx= share->keydef + info->lastinx; for (keydef= share->keydef, end= keydef + share->keys; keydef < end; keydef++) { - if (hp_rec_key_cmp(keydef, old, heap_new, 0)) + if (hp_rec_key_cmp(keydef, old_record, new_record, 0)) { - if ((*keydef->delete_key)(info, keydef, old, pos, keydef == p_lastinx) || - (*keydef->write_key)(info, keydef, heap_new, pos)) + if ((*keydef->delete_key)(info, keydef, old_record, pos, keydef == p_lastinx) || + (*keydef->write_key)(info, keydef, new_record, pos)) goto err; if (share->auto_key == (uint) (keydef - share->keydef + 1)) auto_key_changed= 1; } } - memcpy(pos,heap_new,(size_t) share->reclength); + hp_copy_record_data_to_chunkset(share, new_record, pos); if (++(share->records) == share->blength) share->blength+= share->blength; + if (new_chunk_count < old_chunk_count) { + /* Shrink the chunkset to its new size */ + hp_reallocate_chunkset(&share->recordspace, new_chunk_count, pos); + } + #if !defined(DBUG_OFF) && defined(EXTRA_HEAP_DEBUG) DBUG_EXECUTE("check_heap",heap_check_heap(info, 0);); #endif if (auto_key_changed) - heap_update_auto_increment(info, heap_new); + heap_update_auto_increment(info, new_record); DBUG_RETURN(0); err: @@ -63,26 +82,33 @@ int heap_update(HP_INFO *info, const byt if (keydef->algorithm == HA_KEY_ALG_BTREE) { /* we don't need to delete non-inserted key from rb-tree */ - if ((*keydef->write_key)(info, keydef, old, pos)) + if ((*keydef->write_key)(info, keydef, old_record, pos)) { if (++(share->records) == share->blength) - share->blength+= share->blength; + share->blength+= share->blength; DBUG_RETURN(my_errno); } keydef--; } while (keydef >= share->keydef) { - if (hp_rec_key_cmp(keydef, old, heap_new, 0)) + if (hp_rec_key_cmp(keydef, old_record, new_record, 0)) { - if ((*keydef->delete_key)(info, keydef, heap_new, pos, 0) || - (*keydef->write_key)(info, keydef, old, pos)) - break; + if ((*keydef->delete_key)(info, keydef, new_record, pos, 0) || + (*keydef->write_key)(info, keydef, old_record, pos)) + break; } keydef--; } } + if (++(share->records) == share->blength) share->blength+= share->blength; + + if (new_chunk_count > old_chunk_count) { + /* Shrink the chunkset to its original size */ + hp_reallocate_chunkset(&share->recordspace, old_chunk_count, pos); + } + DBUG_RETURN(my_errno); } /* heap_update */ diff -upraN D:\projects\mysql\mysql-5.0.45-orig/heap/hp_write.c ./heap/hp_write.c --- D:\projects\mysql\mysql-5.0.45-orig/heap/hp_write.c 2007-07-04 06:06:42.000000000 -0700 +++ ./heap/hp_write.c 2008-02-28 21:27:44.000000000 -0800 @@ -25,23 +25,34 @@ #define HIGHFIND 4 #define HIGHUSED 8 -static byte *next_free_record_pos(HP_SHARE *info); static HASH_INFO *hp_find_free_hash(HP_SHARE *info, HP_BLOCK *block, - ulong records); + ulong records); int heap_write(HP_INFO *info, const byte *record) { HP_KEYDEF *keydef, *end; byte *pos; HP_SHARE *share=info->s; + uint rec_length, chunk_count; + DBUG_ENTER("heap_write"); + #ifndef DBUG_OFF if (info->mode & O_RDONLY) { DBUG_RETURN(my_errno=EACCES); } #endif - if (!(pos=next_free_record_pos(share))) + + if ((share->records >= share->max_records && share->max_records) || + (share->recordspace.total_data_length + share->index_length >= share->max_table_size)) + { + DBUG_RETURN(my_errno=HA_ERR_RECORD_FILE_FULL); + } + + rec_length = hp_get_encoded_data_length(share, record, &chunk_count); + + if (!(pos=hp_allocate_chunkset(&share->recordspace, chunk_count))) DBUG_RETURN(my_errno); share->changed=1; @@ -52,10 +63,11 @@ int heap_write(HP_INFO *info, const byte goto err; } - memcpy(pos,record,(size_t) share->reclength); - pos[share->reclength]=1; /* Mark record as not deleted */ + hp_copy_record_data_to_chunkset(share, record, pos); + if (++share->records == share->blength) share->blength+= share->blength; + info->current_ptr=pos; info->current_hash_ptr=0; info->update|=HA_STATE_AKTIV; @@ -87,10 +99,7 @@ err: keydef--; } - share->deleted++; - *((byte**) pos)=share->del_link; - share->del_link=pos; - pos[share->reclength]=0; /* Record deleted */ + hp_free_chunks(&share->recordspace, pos); DBUG_RETURN(my_errno); } /* heap_write */ @@ -128,43 +137,6 @@ int hp_rb_write_key(HP_INFO *info, HP_KE return 0; } - /* Find where to place new record */ - -static byte *next_free_record_pos(HP_SHARE *info) -{ - int block_pos; - byte *pos; - ulong length; - DBUG_ENTER("next_free_record_pos"); - - if (info->del_link) - { - pos=info->del_link; - info->del_link= *((byte**) pos); - info->deleted--; - DBUG_PRINT("exit",("Used old position: 0x%lx",(long) pos)); - DBUG_RETURN(pos); - } - if (!(block_pos=(info->records % info->block.records_in_block))) - { - if ((info->records > info->max_records && info->max_records) || - (info->data_length + info->index_length >= info->max_table_size)) - { - my_errno=HA_ERR_RECORD_FILE_FULL; - DBUG_RETURN(NULL); - } - if (hp_get_new_block(&info->block,&length)) - DBUG_RETURN(NULL); - info->data_length+=length; - } - DBUG_PRINT("exit",("Used new position: 0x%lx", - (long) ((byte*) info->block.level_info[0].last_blocks+ - block_pos * info->block.recbuffer))); - DBUG_RETURN((byte*) info->block.level_info[0].last_blocks+ - block_pos*info->block.recbuffer); -} - - /* Write a hash-key to the hash-index SYNOPSIS diff -upraN D:\projects\mysql\mysql-5.0.45-orig/include/heap.h ./include/heap.h --- D:\projects\mysql\mysql-5.0.45-orig/include/heap.h 2007-07-04 06:06:26.000000000 -0700 +++ ./include/heap.h 2008-02-28 21:10:34.000000000 -0800 @@ -129,22 +129,47 @@ typedef struct st_hp_keydef /* Key defi uint (*get_key_length)(struct st_hp_keydef *keydef, const byte *key); } HP_KEYDEF; -typedef struct st_heap_share +typedef struct st_heap_columndef /* column information */ +{ + int16 type; /* en_fieldtype */ + uint32 length; /* length of field */ + uint32 offset; /* Offset to position in row */ + uint8 null_bit; /* If column may be 0 */ + uint16 null_pos; /* position for null marker */ + uint8 length_bytes; /* length of the size, 1 o 2 bytes */ +} HP_COLUMNDEF; + +typedef struct st_heap_dataspace /* control data for data space */ { HP_BLOCK block; + uint chunk_count; /* Total chunks ever allocated in this dataspace */ + uint del_chunk_count; /* Deleted chunks count */ + byte *del_link; /* Link to last deleted chunk */ + uint chunk_length; /* Total length of one chunk */ + uint chunk_dataspace_length; /* Length of payload that will be placed into one chunk */ + uint offset_status; /* Offset of the status flag relative to the chunk start */ + uint offset_link; /* Offset of the linking pointer relative to the chunk start */ + uint is_variable_size; /* Test whether records have variable size and so "next" pointer */ + ulonglong total_data_length; /* Total size allocated within this data space */ +} HP_DATASPACE; + +typedef struct st_heap_share +{ HP_KEYDEF *keydef; + HP_COLUMNDEF *column_defs; + HP_DATASPACE recordspace; /* Describes "block", which contains actual records */ ulong min_records,max_records; /* Params to open */ - ulonglong data_length,index_length,max_table_size; + ulonglong index_length,max_table_size; uint key_stat_version; /* version to indicate insert/delete */ - uint records; /* records */ - uint blength; /* records rounded up to 2^n */ - uint deleted; /* Deleted records in database */ - uint reclength; /* Length of one record */ + uint records; /* Actual record (row) count */ + uint blength; /* used_chunk_count rounded up to 2^n */ + uint fixed_data_length; /* Length of record's fixed part, which contains keys and always fits into the first chunk */ + uint fixed_column_count; /* Number of columns stored in fixed_data_length */ uint changed; uint keys,max_key_length; + uint column_count; uint currently_disabled_keys; /* saved value from "keys" when disabled */ uint open_count; - byte *del_link; /* Link to next block with del. rec */ my_string name; /* Name of "memory-file" */ #ifdef THREAD THR_LOCK lock; @@ -186,6 +211,8 @@ typedef struct st_heap_create_info { uint auto_key; /* keynr [1 - maxkey] for auto key */ uint auto_key_type; + uint max_chunk_size; + uint is_dynamic; ulonglong max_table_size; ulonglong auto_increment; my_bool with_auto_increment; @@ -195,15 +222,18 @@ typedef struct st_heap_create_info extern HP_INFO *heap_open(const char *name, int mode); extern int heap_close(HP_INFO *info); -extern int heap_write(HP_INFO *info,const byte *buff); -extern int heap_update(HP_INFO *info,const byte *old,const byte *newdata); +extern int heap_write(HP_INFO *info, const byte *record); +extern int heap_update(HP_INFO *info, const byte *old_record, const byte *new_record); extern int heap_rrnd(HP_INFO *info,byte *buf,byte *pos); extern int heap_scan_init(HP_INFO *info); extern int heap_scan(register HP_INFO *info, byte *record); extern int heap_delete(HP_INFO *info,const byte *buff); extern int heap_info(HP_INFO *info,HEAPINFO *x,int flag); extern int heap_create(const char *name, uint keys, HP_KEYDEF *keydef, - uint reclength, ulong max_records, ulong min_records, + uint columns, HP_COLUMNDEF *columndef, + uint max_key_fieldnr, uint key_part_size, + uint reclength, uint keys_memory_size, + ulong max_records, ulong min_records, HP_CREATE_INFO *create_info); extern int heap_delete_table(const char *name); extern int heap_extra(HP_INFO *info,enum ha_extra_function function); diff -upraN D:\projects\mysql\mysql-5.0.45-orig/mysql-test/r/heap_var.result ./mysql-test/r/heap_var.result --- D:\projects\mysql\mysql-5.0.45-orig/mysql-test/r/heap_var.result 1969-12-31 16:00:00.000000000 -0800 +++ ./mysql-test/r/heap_var.result 2008-03-04 13:27:42.000000000 -0800 @@ -0,0 +1,79 @@ +drop table if exists t1; +set @@session.max_heap_table_size=16*1024*1024; +create table t1 (a int not null, b varchar(400), c int, primary key (a), key (c)) engine=heap comment="testing heaps" block_size=128; +ERROR 42000: Incorrect usage/placement of 'block_size' +create table t1 (a int not null, b int, c varchar(400), primary key (a), key (b)) engine=heap comment="testing heaps" block_size=4; +ERROR 42000: Incorrect usage/placement of 'block_size' +create table t1 (a int not null, b int, c varchar(400), d varchar(400), primary key (a), key (b)) engine=heap comment="testing heaps" block_size=24; +insert into t1 values (1,1,'012',NULL), (2,2,'0123456789',NULL), (3,3,'012345678901234567890123456789',NULL), (4,4,NULL,'0123456789012345678901234567890123456789012345678901234567890123456789'); +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +3 3 012345678901234567890123456789 NULL +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +delete from t1 where a = 3; +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +insert into t1 values (5,5,NULL,'0123'), (6,6,NULL,'0123'); +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +6 6 NULL 0123 +5 5 NULL 0123 +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +update t1 set c = '012345678901234567890123456789' where a = 2; +select * from t1; +a b c d +1 1 012 NULL +2 2 012345678901234567890123456789 NULL +6 6 NULL 0123 +5 5 NULL 0123 +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +update t1 set c = '0123456789' where a = 2; +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +6 6 NULL 0123 +5 5 NULL 0123 +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +insert into t1 values (7,7,'0123',NULL), (8,8,'0123',NULL); +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +6 6 NULL 0123 +5 5 NULL 0123 +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +7 7 0123 NULL +8 8 0123 NULL +alter table t1 block_size = 0; +alter table t1 row_format = dynamic; +alter table t1 block_size = 128, max_rows = 10001; +select * from t1; +a b c d +1 1 012 NULL +2 2 0123456789 NULL +6 6 NULL 0123 +5 5 NULL 0123 +4 4 NULL 0123456789012345678901234567890123456789012345678901234567890123456789 +7 7 0123 NULL +8 8 0123 NULL +delete from t1; +select * from t1; +a b c d +select count(*) from t1; +count(*) +10001 +insert into t1 values (100000,100000,NULL,'0123'), (100000,100000,NULL,'0123'); +ERROR HY000: The table 't1' is full +select count(*) from t1; +count(*) +10001 +set @@session.max_heap_table_size=default; +drop table t1; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/mysql-test/t/heap_var.test ./mysql-test/t/heap_var.test --- D:\projects\mysql\mysql-5.0.45-orig/mysql-test/t/heap_var.test 1969-12-31 16:00:00.000000000 -0800 +++ ./mysql-test/t/heap_var.test 2008-03-04 13:27:24.000000000 -0800 @@ -0,0 +1,78 @@ +# +# Test heap tables with variable-sized records. +# + +--disable_warnings +drop table if exists t1; +--enable_warnings + +set @@session.max_heap_table_size=16*1024*1024; + +--error 1234 +create table t1 (a int not null, b varchar(400), c int, primary key (a), key (c)) engine=heap comment="testing heaps" block_size=128; + +--error 1234 +create table t1 (a int not null, b int, c varchar(400), primary key (a), key (b)) engine=heap comment="testing heaps" block_size=4; + +create table t1 (a int not null, b int, c varchar(400), d varchar(400), primary key (a), key (b)) engine=heap comment="testing heaps" block_size=24; + +#show table status like "t1"; + +insert into t1 values (1,1,'012',NULL), (2,2,'0123456789',NULL), (3,3,'012345678901234567890123456789',NULL), (4,4,NULL,'0123456789012345678901234567890123456789012345678901234567890123456789'); +select * from t1; + +delete from t1 where a = 3; +select * from t1; + +insert into t1 values (5,5,NULL,'0123'), (6,6,NULL,'0123'); +select * from t1; + +update t1 set c = '012345678901234567890123456789' where a = 2; +select * from t1; + +update t1 set c = '0123456789' where a = 2; +select * from t1; + +insert into t1 values (7,7,'0123',NULL), (8,8,'0123',NULL); +select * from t1; + +#show table status like "t1"; +alter table t1 block_size = 0; +#show table status like "t1"; +alter table t1 row_format = dynamic; +#show table status like "t1"; +alter table t1 block_size = 128, max_rows = 10001; +#show table status like "t1"; + +select * from t1; + +delete from t1; +select * from t1; + +let $1=10001; + +disable_query_log; + +while ($1) +{ + + eval insert into t1 values ($1,$1,$1,$1); + + dec $1; + +} +enable_query_log; + +select count(*) from t1; + +--error 1114 +insert into t1 values (100000,100000,NULL,'0123'), (100000,100000,NULL,'0123'); + +#show table status like "t1"; +select count(*) from t1; + +set @@session.max_heap_table_size=default; + +drop table t1; + +# End of 5.0 tests diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/ha_heap.cc ./sql/ha_heap.cc --- D:\projects\mysql\mysql-5.0.45-orig/sql/ha_heap.cc 2007-07-04 06:06:42.000000000 -0700 +++ ./sql/ha_heap.cc 2008-02-29 16:13:12.000000000 -0800 @@ -356,6 +356,14 @@ int ha_heap::info(uint flag) return 0; } +enum row_type ha_heap::get_row_type() const +{ + if (file->s->recordspace.is_variable_size) + return ROW_TYPE_DYNAMIC; + + return ROW_TYPE_FIXED; +} + int ha_heap::extra(enum ha_extra_function operation) { return heap_extra(file,operation); @@ -543,9 +551,11 @@ ha_rows ha_heap::records_in_range(uint i int ha_heap::create(const char *name, TABLE *table_arg, HA_CREATE_INFO *create_info) { - uint key, parts, mem_per_row= 0, keys= table_arg->s->keys; + uint key, parts, mem_per_row_keys= 0, keys= table_arg->s->keys; uint auto_key= 0, auto_key_type= 0; - ha_rows max_rows; + uint max_key_fieldnr = 0, key_part_size = 0; + uint column_idx, column_count= table_arg->s->fields; + HP_COLUMNDEF *columndef; HP_KEYDEF *keydef; HA_KEYSEG *seg; char buff[FN_REFLEN]; @@ -553,13 +563,49 @@ int ha_heap::create(const char *name, TA TABLE_SHARE *share= table_arg->s; bool found_real_auto_increment= 0; + if (!(columndef= (HP_COLUMNDEF*) my_malloc(column_count * sizeof(HP_COLUMNDEF), MYF(MY_WME)))) + return my_errno; + + for (column_idx= 0; column_idx < column_count; column_idx++) + { + Field* field= *(table_arg->field + column_idx); + HP_COLUMNDEF* column= columndef + column_idx; + column->type= (uint16)field->type(); + column->length= field->pack_length(); + column->offset= field->offset(); + + if (field->null_bit) + { + column->null_bit= field->null_bit; + column->null_pos= (uint) (field->null_ptr - (uchar*) table_arg->record[0]); + } + else + { + column->null_bit= 0; + column->null_pos= 0; + } + + if (field->type() == MYSQL_TYPE_VARCHAR) + { + column->length_bytes= (uint8)(((Field_varstring*)field)->length_bytes); + } + else + { + column->length_bytes= 0; + } + } + for (key= parts= 0; key < keys; key++) parts+= table_arg->key_info[key].key_parts; if (!(keydef= (HP_KEYDEF*) my_malloc(keys * sizeof(HP_KEYDEF) + parts * sizeof(HA_KEYSEG), MYF(MY_WME)))) + { + my_free((gptr) columndef, MYF(0)); return my_errno; + } + seg= my_reinterpret_cast(HA_KEYSEG*) (keydef + keys); for (key= 0; key < keys; key++) { @@ -575,11 +621,11 @@ int ha_heap::create(const char *name, TA case HA_KEY_ALG_UNDEF: case HA_KEY_ALG_HASH: keydef[key].algorithm= HA_KEY_ALG_HASH; - mem_per_row+= sizeof(char*) * 2; // = sizeof(HASH_INFO) + mem_per_row_keys+= sizeof(char*) * 2; // = sizeof(HASH_INFO) break; case HA_KEY_ALG_BTREE: keydef[key].algorithm= HA_KEY_ALG_BTREE; - mem_per_row+=sizeof(TREE_ELEMENT)+pos->key_length+sizeof(char*); + mem_per_row_keys+=sizeof(TREE_ELEMENT)+pos->key_length+sizeof(char*); break; default: DBUG_ASSERT(0); // cannot happen @@ -604,14 +650,18 @@ int ha_heap::create(const char *name, TA seg->length= (uint) key_part->length; seg->flag= key_part->key_part_flag; + if ((seg->start + seg->length) > key_part_size) { + key_part_size= seg->start + seg->length; + } + if (field->flags & (ENUM_FLAG | SET_FLAG)) seg->charset= &my_charset_bin; else seg->charset= field->charset(); if (field->null_ptr) { - seg->null_bit= field->null_bit; - seg->null_pos= (uint) (field->null_ptr - (uchar*) table_arg->record[0]); + seg->null_bit= field->null_bit; + seg->null_pos= (uint) (field->null_ptr - (uchar*) table_arg->record[0]); } else { @@ -629,11 +679,20 @@ int ha_heap::create(const char *name, TA auto_key= key+ 1; auto_key_type= field->key_type(); } + + if (key_part->fieldnr > max_key_fieldnr) + { + max_key_fieldnr= key_part->fieldnr; + } } } - mem_per_row+= MY_ALIGN(share->reclength + 1, sizeof(char*)); - max_rows = (ha_rows) (table->in_use->variables.max_heap_table_size / - (ulonglong) mem_per_row); + + if (key_part_size < share->null_bytes + ((share->last_null_bit_pos+7) >> 3)) + { + /* Make sure to include null fields regardless of the presense of keys */ + key_part_size = share->null_bytes + ((share->last_null_bit_pos+7) >> 3); + } + if (table_arg->found_next_number_field) { keydef[share->next_number_index].flag|= HA_AUTO_KEY; @@ -646,15 +705,18 @@ int ha_heap::create(const char *name, TA create_info->auto_increment_value - 1 : 0); hp_create_info.max_table_size=current_thd->variables.max_heap_table_size; hp_create_info.with_auto_increment= found_real_auto_increment; - max_rows = (ha_rows) (hp_create_info.max_table_size / mem_per_row); + hp_create_info.max_chunk_size= table->s->block_size; + hp_create_info.is_dynamic= (table->s->row_type == ROW_TYPE_DYNAMIC); error= heap_create(fn_format(buff,name,"","", MY_REPLACE_EXT|MY_UNPACK_FILENAME), - keys, keydef, share->reclength, - (ulong) ((share->max_rows < max_rows && - share->max_rows) ? - share->max_rows : max_rows), - (ulong) share->min_rows, &hp_create_info); + keys, keydef, + column_count, columndef, + max_key_fieldnr, key_part_size, + share->reclength, mem_per_row_keys, + (ulong) share->max_rows, (ulong) share->min_rows, + &hp_create_info); my_free((gptr) keydef, MYF(0)); + my_free((gptr) columndef, MYF(0)); if (file) info(HA_STATUS_NO_LOCK | HA_STATUS_CONST | HA_STATUS_VARIABLE); return (error); @@ -666,6 +728,13 @@ void ha_heap::update_create_info(HA_CREA table->file->info(HA_STATUS_AUTO); if (!(create_info->used_fields & HA_CREATE_USED_AUTO)) create_info->auto_increment_value= auto_increment_value; + if (!(create_info->used_fields & HA_CREATE_USED_BLOCK_SIZE)) + { + if (file->s->recordspace.is_variable_size) + create_info->block_size= file->s->recordspace.chunk_length; + else + create_info->block_size= 0; + } } ulonglong ha_heap::get_auto_increment() diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/ha_heap.h ./sql/ha_heap.h --- D:\projects\mysql\mysql-5.0.45-orig/sql/ha_heap.h 2007-07-04 06:06:50.000000000 -0700 +++ ./sql/ha_heap.h 2008-02-28 21:13:22.000000000 -0800 @@ -42,8 +42,7 @@ public: return ((table->key_info[inx].algorithm == HA_KEY_ALG_BTREE) ? "BTREE" : "HASH"); } - /* Rows also use a fixed-size format */ - enum row_type get_row_type() const { return ROW_TYPE_FIXED; } + enum row_type get_row_type() const; const char **bas_ext() const; ulong table_flags() const { diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/handler.h ./sql/handler.h --- D:\projects\mysql\mysql-5.0.45-orig/sql/handler.h 2007-07-04 06:06:26.000000000 -0700 +++ ./sql/handler.h 2008-02-28 21:13:10.000000000 -0800 @@ -216,6 +216,7 @@ enum row_type { ROW_TYPE_NOT_USED=-1, RO #define HA_CREATE_USED_COMMENT (1L << 16) #define HA_CREATE_USED_PASSWORD (1L << 17) #define HA_CREATE_USED_CONNECTION (1L << 18) +#define HA_CREATE_USED_BLOCK_SIZE (1L << 19) typedef ulonglong my_xid; // this line is the same as in log_event.h #define MYSQL_XID_PREFIX "MySQLXid" @@ -438,6 +439,7 @@ typedef struct st_ha_create_information ulong avg_row_length; ulong raid_chunksize; ulong used_fields; + ulong block_size; SQL_LIST merge_list; enum db_type db_type; enum row_type row_type; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/lex.h ./sql/lex.h --- D:\projects\mysql\mysql-5.0.45-orig/sql/lex.h 2007-07-04 06:06:42.000000000 -0700 +++ ./sql/lex.h 2008-02-27 14:23:12.000000000 -0800 @@ -88,6 +88,7 @@ static SYMBOL symbols[] = { { "BIT", SYM(BIT_SYM)}, { "BLOB", SYM(BLOB_SYM)}, { "BLOCK", SYM(BLOCK_SYM)}, + { "BLOCK_SIZE", SYM(BLOCK_SIZE_SYM)}, { "BOOL", SYM(BOOL_SYM)}, { "BOOLEAN", SYM(BOOLEAN_SYM)}, { "BOTH", SYM(BOTH)}, diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/sql_show.cc ./sql/sql_show.cc --- D:\projects\mysql\mysql-5.0.45-orig/sql/sql_show.cc 2007-07-04 06:06:16.000000000 -0700 +++ ./sql/sql_show.cc 2008-02-26 16:18:12.000000000 -0800 @@ -1096,6 +1096,13 @@ store_create_info(THD *thd, TABLE_LIST * packet->append(STRING_WITH_LEN(" ROW_FORMAT=")); packet->append(ha_row_type[(uint) share->row_type]); } + if (share->block_size) + { + char *end; + packet->append(STRING_WITH_LEN(" BLOCK_SIZE=")); + end= longlong10_to_str(share->block_size, buff,10); + packet->append(buff, (uint) (end - buff)); + } table->file->append_create_info(packet); if (share->comment.length) { @@ -2564,7 +2571,7 @@ static int get_schema_tables_record(THD table->field[18]->set_notnull(); } - char option_buff[350],*ptr; + char option_buff[400],*ptr; ptr=option_buff; if (share->min_rows) { @@ -2593,6 +2600,11 @@ static int get_schema_tables_record(THD ptr=strxmov(ptr, " row_format=", ha_row_type[(uint) share->row_type], NullS); + if (share->block_size) + { + ptr= strmov(ptr, " block_size="); + ptr= longlong10_to_str(share->block_size, ptr, 10); + } if (file->raid_type) { char buff[100]; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/sql_table.cc ./sql/sql_table.cc --- D:\projects\mysql\mysql-5.0.45-orig/sql/sql_table.cc 2007-07-04 06:06:22.000000000 -0700 +++ ./sql/sql_table.cc 2008-02-26 15:59:44.000000000 -0800 @@ -3348,6 +3348,8 @@ view_err: create_info->max_rows= table->s->max_rows; if (!(used_fields & HA_CREATE_USED_AVG_ROW_LENGTH)) create_info->avg_row_length= table->s->avg_row_length; + if (!(used_fields & HA_CREATE_USED_BLOCK_SIZE)) + create_info->block_size= table->s->block_size; if (!(used_fields & HA_CREATE_USED_DEFAULT_CHARSET)) create_info->default_table_charset= table->s->table_charset; if (!(used_fields & HA_CREATE_USED_AUTO) && table->found_next_number_field) diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/sql_yacc.yy ./sql/sql_yacc.yy --- D:\projects\mysql\mysql-5.0.45-orig/sql/sql_yacc.yy 2007-07-04 06:06:26.000000000 -0700 +++ ./sql/sql_yacc.yy 2008-02-28 07:50:46.000000000 -0800 @@ -501,6 +501,7 @@ bool my_yyoverflow(short **a, YYSTYPE ** %token BIT_SYM %token BIT_XOR %token BLOB_SYM +%token BLOCK_SIZE_SYM %token BLOCK_SYM %token BOOLEAN_SYM %token BOOL_SYM @@ -2881,6 +2882,7 @@ create_table_option: | MAX_ROWS opt_equal ulonglong_num { Lex->create_info.max_rows= $3; Lex->create_info.used_fields|= HA_CREATE_USED_MAX_ROWS;} | MIN_ROWS opt_equal ulonglong_num { Lex->create_info.min_rows= $3; Lex->create_info.used_fields|= HA_CREATE_USED_MIN_ROWS;} | AVG_ROW_LENGTH opt_equal ulong_num { Lex->create_info.avg_row_length=$3; Lex->create_info.used_fields|= HA_CREATE_USED_AVG_ROW_LENGTH;} + | BLOCK_SIZE_SYM opt_equal ulong_num { Lex->create_info.block_size= $3; Lex->create_info.used_fields|= HA_CREATE_USED_BLOCK_SIZE;} | PASSWORD opt_equal TEXT_STRING_sys { Lex->create_info.password=$3.str; Lex->create_info.used_fields|= HA_CREATE_USED_PASSWORD; } | COMMENT_SYM opt_equal TEXT_STRING_sys { Lex->create_info.comment=$3; Lex->create_info.used_fields|= HA_CREATE_USED_COMMENT; } | AUTO_INC opt_equal ulonglong_num { Lex->create_info.auto_increment_value=$3; Lex->create_info.used_fields|= HA_CREATE_USED_AUTO;} @@ -8146,6 +8148,7 @@ keyword_sp: | BERKELEY_DB_SYM {} | BINLOG_SYM {} | BIT_SYM {} + | BLOCK_SIZE_SYM {} | BLOCK_SYM {} | BOOL_SYM {} | BOOLEAN_SYM {} diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/table.cc ./sql/table.cc --- D:\projects\mysql\mysql-5.0.45-orig/sql/table.cc 2007-07-04 06:06:06.000000000 -0700 +++ ./sql/table.cc 2008-02-28 16:07:38.000000000 -0800 @@ -172,10 +172,19 @@ int openfrm(THD *thd, const char *name, if (!head[32]) // New frm file in 3.23 { share->avg_row_length= uint4korr(head+34); - share-> row_type= (row_type) head[40]; - share->raid_type= head[41]; - share->raid_chunks= head[42]; - share->raid_chunksize= uint4korr(head+43); + share->row_type= (row_type) head[40]; + if (share->db_type != DB_TYPE_HEAP) + { + share->raid_type= head[41]; + share->raid_chunks= head[42]; + share->raid_chunksize= uint4korr(head+43); + } + else + { + /* So far, this is the only way to avoid new versions of FRM file */ + /* According to Sergei, raid_ should not be used by heap anyway */ + share->block_size= uint4korr(head+43); + } share->table_charset= get_charset((uint) head[38],MYF(0)); null_field_first= 1; } @@ -1460,9 +1469,16 @@ File create_frm(THD *thd, my_string name fileinfo[38]= (create_info->default_table_charset ? create_info->default_table_charset->number : 0); fileinfo[40]= (uchar) create_info->row_type; - fileinfo[41]= (uchar) create_info->raid_type; - fileinfo[42]= (uchar) create_info->raid_chunks; - int4store(fileinfo+43,create_info->raid_chunksize); + if (create_info->db_type != DB_TYPE_HEAP) + { + fileinfo[41]= (uchar) create_info->raid_type; + fileinfo[42]= (uchar) create_info->raid_chunks; + int4store(fileinfo+43,create_info->raid_chunksize); + } + else + { + int4store(fileinfo+43, create_info->block_size); + } int4store(fileinfo+47, key_length); tmp= MYSQL_VERSION_ID; // Store to avoid warning from int4store int4store(fileinfo+51, tmp); @@ -1498,6 +1514,7 @@ void update_create_info_from_table(HA_CR create_info->min_rows= share->min_rows; create_info->table_options= share->db_create_options; create_info->avg_row_length= share->avg_row_length; + create_info->block_size= share->block_size; create_info->row_type= share->row_type; create_info->raid_type= share->raid_type; create_info->raid_chunks= share->raid_chunks; diff -upraN D:\projects\mysql\mysql-5.0.45-orig/sql/table.h ./sql/table.h --- D:\projects\mysql\mysql-5.0.45-orig/sql/table.h 2007-07-04 06:06:10.000000000 -0700 +++ ./sql/table.h 2008-02-26 15:57:58.000000000 -0800 @@ -145,6 +145,7 @@ typedef struct st_table_share key_map keys_in_use; key_map keys_for_keyread; ulong avg_row_length; /* create information */ + ulong block_size; /* create information */ ulong raid_chunksize; ulong version, flush_version, mysql_version; ulong timestamp_offset; /* Set to offset+1 of record */