OpenTTD
fileio.cpp
Go to the documentation of this file.
1 /* $Id: fileio.cpp 26514 2014-04-25 21:29:54Z rubidium $ */
2 
3 /*
4  * This file is part of OpenTTD.
5  * OpenTTD is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, version 2.
6  * OpenTTD is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
7  * See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with OpenTTD. If not, see <http://www.gnu.org/licenses/>.
8  */
9 
12 #include "stdafx.h"
13 #include "fileio_func.h"
14 #include "debug.h"
15 #include "fios.h"
16 #include "string_func.h"
17 #include "tar_type.h"
18 #ifdef WIN32
19 #include <windows.h>
20 # define access _taccess
21 #elif defined(__HAIKU__)
22 #include <Path.h>
23 #include <storage/FindDirectory.h>
24 #else
25 #include <unistd.h>
26 #include <pwd.h>
27 #endif
28 #include <sys/stat.h>
29 #include <algorithm>
30 
31 #ifdef WITH_XDG_BASEDIR
32 #include "basedir.h"
33 #endif
34 
35 #include "safeguards.h"
36 
38 #define FIO_BUFFER_SIZE 512
39 
41 struct Fio {
42  byte *buffer, *buffer_end;
43  size_t pos;
44  FILE *cur_fh;
45  const char *filename;
48  const char *filenames[MAX_FILE_SLOTS];
50 #if defined(LIMITED_FDS)
51  uint open_handles;
52  uint usage_count[MAX_FILE_SLOTS];
53 #endif /* LIMITED_FDS */
54 };
55 
56 static Fio _fio;
57 
59 static bool _do_scan_working_directory = true;
60 
61 extern char *_config_file;
62 extern char *_highscore_file;
63 
68 size_t FioGetPos()
69 {
70  return _fio.pos + (_fio.buffer - _fio.buffer_end);
71 }
72 
78 const char *FioGetFilename(uint8 slot)
79 {
80  return _fio.shortnames[slot];
81 }
82 
88 void FioSeekTo(size_t pos, int mode)
89 {
90  if (mode == SEEK_CUR) pos += FioGetPos();
91  _fio.buffer = _fio.buffer_end = _fio.buffer_start + FIO_BUFFER_SIZE;
92  _fio.pos = pos;
93  if (fseek(_fio.cur_fh, _fio.pos, SEEK_SET) < 0) {
94  DEBUG(misc, 0, "Seeking in %s failed", _fio.filename);
95  }
96 }
97 
98 #if defined(LIMITED_FDS)
99 static void FioRestoreFile(int slot)
100 {
101  /* Do we still have the file open, or should we reopen it? */
102  if (_fio.handles[slot] == NULL) {
103  DEBUG(misc, 6, "Restoring file '%s' in slot '%d' from disk", _fio.filenames[slot], slot);
104  FioOpenFile(slot, _fio.filenames[slot]);
105  }
106  _fio.usage_count[slot]++;
107 }
108 #endif /* LIMITED_FDS */
109 
115 void FioSeekToFile(uint8 slot, size_t pos)
116 {
117  FILE *f;
118 #if defined(LIMITED_FDS)
119  /* Make sure we have this file open */
120  FioRestoreFile(slot);
121 #endif /* LIMITED_FDS */
122  f = _fio.handles[slot];
123  assert(f != NULL);
124  _fio.cur_fh = f;
125  _fio.filename = _fio.filenames[slot];
126  FioSeekTo(pos, SEEK_SET);
127 }
128 
134 {
135  if (_fio.buffer == _fio.buffer_end) {
136  _fio.buffer = _fio.buffer_start;
137  size_t size = fread(_fio.buffer, 1, FIO_BUFFER_SIZE, _fio.cur_fh);
138  _fio.pos += size;
139  _fio.buffer_end = _fio.buffer_start + size;
140 
141  if (size == 0) return 0;
142  }
143  return *_fio.buffer++;
144 }
145 
150 void FioSkipBytes(int n)
151 {
152  for (;;) {
153  int m = min(_fio.buffer_end - _fio.buffer, n);
154  _fio.buffer += m;
155  n -= m;
156  if (n == 0) break;
157  FioReadByte();
158  n--;
159  }
160 }
161 
166 uint16 FioReadWord()
167 {
168  byte b = FioReadByte();
169  return (FioReadByte() << 8) | b;
170 }
171 
176 uint32 FioReadDword()
177 {
178  uint b = FioReadWord();
179  return (FioReadWord() << 16) | b;
180 }
181 
187 void FioReadBlock(void *ptr, size_t size)
188 {
189  FioSeekTo(FioGetPos(), SEEK_SET);
190  _fio.pos += fread(ptr, 1, size, _fio.cur_fh);
191 }
192 
197 static inline void FioCloseFile(int slot)
198 {
199  if (_fio.handles[slot] != NULL) {
200  fclose(_fio.handles[slot]);
201 
202  free(_fio.shortnames[slot]);
203  _fio.shortnames[slot] = NULL;
204 
205  _fio.handles[slot] = NULL;
206 #if defined(LIMITED_FDS)
207  _fio.open_handles--;
208 #endif /* LIMITED_FDS */
209  }
210 }
211 
214 {
215  for (int i = 0; i != lengthof(_fio.handles); i++) {
216  FioCloseFile(i);
217  }
218 }
219 
220 #if defined(LIMITED_FDS)
221 static void FioFreeHandle()
222 {
223  /* If we are about to open a file that will exceed the limit, close a file */
224  if (_fio.open_handles + 1 == LIMITED_FDS) {
225  uint i, count;
226  int slot;
227 
228  count = UINT_MAX;
229  slot = -1;
230  /* Find the file that is used the least */
231  for (i = 0; i < lengthof(_fio.handles); i++) {
232  if (_fio.handles[i] != NULL && _fio.usage_count[i] < count) {
233  count = _fio.usage_count[i];
234  slot = i;
235  }
236  }
237  assert(slot != -1);
238  DEBUG(misc, 6, "Closing filehandler '%s' in slot '%d' because of fd-limit", _fio.filenames[slot], slot);
239  FioCloseFile(slot);
240  }
241 }
242 #endif /* LIMITED_FDS */
243 
250 void FioOpenFile(int slot, const char *filename, Subdirectory subdir)
251 {
252  FILE *f;
253 
254 #if defined(LIMITED_FDS)
255  FioFreeHandle();
256 #endif /* LIMITED_FDS */
257  f = FioFOpenFile(filename, "rb", subdir);
258  if (f == NULL) usererror("Cannot open file '%s'", filename);
259  long pos = ftell(f);
260  if (pos < 0) usererror("Cannot read file '%s'", filename);
261 
262  FioCloseFile(slot); // if file was opened before, close it
263  _fio.handles[slot] = f;
264  _fio.filenames[slot] = filename;
265 
266  /* Store the filename without path and extension */
267  const char *t = strrchr(filename, PATHSEPCHAR);
268  _fio.shortnames[slot] = stredup(t == NULL ? filename : t);
269  char *t2 = strrchr(_fio.shortnames[slot], '.');
270  if (t2 != NULL) *t2 = '\0';
271  strtolower(_fio.shortnames[slot]);
272 
273 #if defined(LIMITED_FDS)
274  _fio.usage_count[slot] = 0;
275  _fio.open_handles++;
276 #endif /* LIMITED_FDS */
277  FioSeekToFile(slot, (uint32)pos);
278 }
279 
280 static const char * const _subdirs[] = {
281  "",
282  "save" PATHSEP,
283  "save" PATHSEP "autosave" PATHSEP,
284  "scenario" PATHSEP,
285  "scenario" PATHSEP "heightmap" PATHSEP,
286  "gm" PATHSEP,
287  "data" PATHSEP,
288  "baseset" PATHSEP,
289  "newgrf" PATHSEP,
290  "lang" PATHSEP,
291  "ai" PATHSEP,
292  "ai" PATHSEP "library" PATHSEP,
293  "game" PATHSEP,
294  "game" PATHSEP "library" PATHSEP,
295  "screenshot" PATHSEP,
296 };
297 assert_compile(lengthof(_subdirs) == NUM_SUBDIRS);
298 
299 const char *_searchpaths[NUM_SEARCHPATHS];
300 TarList _tar_list[NUM_SUBDIRS];
301 TarFileList _tar_filelist[NUM_SUBDIRS];
302 
303 typedef std::map<std::string, std::string> TarLinkList;
304 static TarLinkList _tar_linklist[NUM_SUBDIRS];
305 
312 bool FioCheckFileExists(const char *filename, Subdirectory subdir)
313 {
314  FILE *f = FioFOpenFile(filename, "rb", subdir);
315  if (f == NULL) return false;
316 
317  FioFCloseFile(f);
318  return true;
319 }
320 
326 bool FileExists(const char *filename)
327 {
328 #if defined(WINCE)
329  /* There is always one platform that doesn't support basic commands... */
330  HANDLE hand = CreateFile(OTTD2FS(filename), 0, 0, NULL, OPEN_EXISTING, 0, NULL);
331  if (hand == INVALID_HANDLE_VALUE) return 1;
332  CloseHandle(hand);
333  return 0;
334 #else
335  return access(OTTD2FS(filename), 0) == 0;
336 #endif
337 }
338 
342 void FioFCloseFile(FILE *f)
343 {
344  fclose(f);
345 }
346 
347 char *FioGetFullPath(char *buf, const char *last, Searchpath sp, Subdirectory subdir, const char *filename)
348 {
349  assert(subdir < NUM_SUBDIRS);
350  assert(sp < NUM_SEARCHPATHS);
351 
352  seprintf(buf, last, "%s%s%s", _searchpaths[sp], _subdirs[subdir], filename);
353  return buf;
354 }
355 
364 char *FioFindFullPath(char *buf, const char *last, Subdirectory subdir, const char *filename)
365 {
366  Searchpath sp;
367  assert(subdir < NUM_SUBDIRS);
368 
369  FOR_ALL_SEARCHPATHS(sp) {
370  FioGetFullPath(buf, last, sp, subdir, filename);
371  if (FileExists(buf)) return buf;
372 #if !defined(WIN32)
373  /* Be, as opening files, aware that sometimes the filename
374  * might be in uppercase when it is in lowercase on the
375  * disk. Of course Windows doesn't care about casing. */
376  if (strtolower(buf + strlen(_searchpaths[sp]) - 1) && FileExists(buf)) return buf;
377 #endif
378  }
379 
380  return NULL;
381 }
382 
383 char *FioAppendDirectory(char *buf, const char *last, Searchpath sp, Subdirectory subdir)
384 {
385  assert(subdir < NUM_SUBDIRS);
386  assert(sp < NUM_SEARCHPATHS);
387 
388  seprintf(buf, last, "%s%s", _searchpaths[sp], _subdirs[subdir]);
389  return buf;
390 }
391 
392 char *FioGetDirectory(char *buf, const char *last, Subdirectory subdir)
393 {
394  Searchpath sp;
395 
396  /* Find and return the first valid directory */
397  FOR_ALL_SEARCHPATHS(sp) {
398  char *ret = FioAppendDirectory(buf, last, sp, subdir);
399  if (FileExists(buf)) return ret;
400  }
401 
402  /* Could not find the directory, fall back to a base path */
403  strecpy(buf, _personal_dir, last);
404 
405  return buf;
406 }
407 
408 static FILE *FioFOpenFileSp(const char *filename, const char *mode, Searchpath sp, Subdirectory subdir, size_t *filesize)
409 {
410 #if defined(WIN32) && defined(UNICODE)
411  /* fopen is implemented as a define with ellipses for
412  * Unicode support (prepend an L). As we are not sending
413  * a string, but a variable, it 'renames' the variable,
414  * so make that variable to makes it compile happily */
415  wchar_t Lmode[5];
416  MultiByteToWideChar(CP_ACP, 0, mode, -1, Lmode, lengthof(Lmode));
417 #endif
418  FILE *f = NULL;
419  char buf[MAX_PATH];
420 
421  if (subdir == NO_DIRECTORY) {
422  strecpy(buf, filename, lastof(buf));
423  } else {
424  seprintf(buf, lastof(buf), "%s%s%s", _searchpaths[sp], _subdirs[subdir], filename);
425  }
426 
427 #if defined(WIN32)
428  if (mode[0] == 'r' && GetFileAttributes(OTTD2FS(buf)) == INVALID_FILE_ATTRIBUTES) return NULL;
429 #endif
430 
431  f = fopen(buf, mode);
432 #if !defined(WIN32)
433  if (f == NULL && strtolower(buf + ((subdir == NO_DIRECTORY) ? 0 : strlen(_searchpaths[sp]) - 1))) {
434  f = fopen(buf, mode);
435  }
436 #endif
437  if (f != NULL && filesize != NULL) {
438  /* Find the size of the file */
439  fseek(f, 0, SEEK_END);
440  *filesize = ftell(f);
441  fseek(f, 0, SEEK_SET);
442  }
443  return f;
444 }
445 
453 FILE *FioFOpenFileTar(TarFileListEntry *entry, size_t *filesize)
454 {
455  FILE *f = fopen(entry->tar_filename, "rb");
456  if (f == NULL) return f;
457 
458  if (fseek(f, entry->position, SEEK_SET) < 0) {
459  fclose(f);
460  return NULL;
461  }
462 
463  if (filesize != NULL) *filesize = entry->size;
464  return f;
465 }
466 
474 FILE *FioFOpenFile(const char *filename, const char *mode, Subdirectory subdir, size_t *filesize)
475 {
476  FILE *f = NULL;
477  Searchpath sp;
478 
479  assert(subdir < NUM_SUBDIRS || subdir == NO_DIRECTORY);
480 
481  FOR_ALL_SEARCHPATHS(sp) {
482  f = FioFOpenFileSp(filename, mode, sp, subdir, filesize);
483  if (f != NULL || subdir == NO_DIRECTORY) break;
484  }
485 
486  /* We can only use .tar in case of data-dir, and read-mode */
487  if (f == NULL && mode[0] == 'r' && subdir != NO_DIRECTORY) {
488  static const uint MAX_RESOLVED_LENGTH = 2 * (100 + 100 + 155) + 1; // Enough space to hold two filenames plus link. See 'TarHeader'.
489  char resolved_name[MAX_RESOLVED_LENGTH];
490 
491  /* Filenames in tars are always forced to be lowercase */
492  strecpy(resolved_name, filename, lastof(resolved_name));
493  strtolower(resolved_name);
494 
495  size_t resolved_len = strlen(resolved_name);
496 
497  /* Resolve ONE directory link */
498  for (TarLinkList::iterator link = _tar_linklist[subdir].begin(); link != _tar_linklist[subdir].end(); link++) {
499  const std::string &src = link->first;
500  size_t len = src.length();
501  if (resolved_len >= len && resolved_name[len - 1] == PATHSEPCHAR && strncmp(src.c_str(), resolved_name, len) == 0) {
502  /* Apply link */
503  char resolved_name2[MAX_RESOLVED_LENGTH];
504  const std::string &dest = link->second;
505  strecpy(resolved_name2, &(resolved_name[len]), lastof(resolved_name2));
506  strecpy(resolved_name, dest.c_str(), lastof(resolved_name));
507  strecpy(&(resolved_name[dest.length()]), resolved_name2, lastof(resolved_name));
508  break; // Only resolve one level
509  }
510  }
511 
512  TarFileList::iterator it = _tar_filelist[subdir].find(resolved_name);
513  if (it != _tar_filelist[subdir].end()) {
514  f = FioFOpenFileTar(&((*it).second), filesize);
515  }
516  }
517 
518  /* Sometimes a full path is given. To support
519  * the 'subdirectory' must be 'removed'. */
520  if (f == NULL && subdir != NO_DIRECTORY) {
521  switch (subdir) {
522  case BASESET_DIR:
523  f = FioFOpenFile(filename, mode, OLD_GM_DIR, filesize);
524  if (f != NULL) break;
525  /* FALL THROUGH */
526  case NEWGRF_DIR:
527  f = FioFOpenFile(filename, mode, OLD_DATA_DIR, filesize);
528  break;
529 
530  default:
531  f = FioFOpenFile(filename, mode, NO_DIRECTORY, filesize);
532  break;
533  }
534  }
535 
536  return f;
537 }
538 
543 static void FioCreateDirectory(const char *name)
544 {
545  /* Ignore directory creation errors; they'll surface later on, and most
546  * of the time they are 'directory already exists' errors anyhow. */
547 #if defined(WIN32) || defined(WINCE)
548  CreateDirectory(OTTD2FS(name), NULL);
549 #elif defined(OS2) && !defined(__INNOTEK_LIBC__)
550  mkdir(OTTD2FS(name));
551 #elif defined(__MORPHOS__) || defined(__AMIGAOS__)
552  char buf[MAX_PATH];
553  strecpy(buf, name, lastof(buf));
554 
555  size_t len = strlen(name) - 1;
556  if (buf[len] == '/') {
557  buf[len] = '\0'; // Kill pathsep, so mkdir() will not fail
558  }
559 
560  mkdir(OTTD2FS(buf), 0755);
561 #else
562  mkdir(OTTD2FS(name), 0755);
563 #endif
564 }
565 
573 bool AppendPathSeparator(char *buf, const char *last)
574 {
575  size_t s = strlen(buf);
576 
577  /* Length of string + path separator + '\0' */
578  if (s != 0 && buf[s - 1] != PATHSEPCHAR) {
579  if (&buf[s] >= last) return false;
580 
581  seprintf(buf + s, last, "%c", PATHSEPCHAR);
582  }
583 
584  return true;
585 }
586 
593 char *BuildWithFullPath(const char *dir)
594 {
595  char *dest = MallocT<char>(MAX_PATH);
596  char *last = dest + MAX_PATH - 1;
597  strecpy(dest, dir, last);
598 
599  /* Check if absolute or relative path */
600  const char *s = strchr(dest, PATHSEPCHAR);
601 
602  /* Add absolute path */
603  if (s == NULL || dest != s) {
604  if (getcwd(dest, MAX_PATH) == NULL) *dest = '\0';
605  AppendPathSeparator(dest, last);
606  strecat(dest, dir, last);
607  }
608  AppendPathSeparator(dest, last);
609 
610  return dest;
611 }
612 
618 const char *FioTarFirstDir(const char *tarname, Subdirectory subdir)
619 {
620  TarList::iterator it = _tar_list[subdir].find(tarname);
621  if (it == _tar_list[subdir].end()) return NULL;
622  return (*it).second.dirname;
623 }
624 
625 static void TarAddLink(const std::string &srcParam, const std::string &destParam, Subdirectory subdir)
626 {
627  std::string src = srcParam;
628  std::string dest = destParam;
629  /* Tar internals assume lowercase */
630  std::transform(src.begin(), src.end(), src.begin(), tolower);
631  std::transform(dest.begin(), dest.end(), dest.begin(), tolower);
632 
633  TarFileList::iterator dest_file = _tar_filelist[subdir].find(dest);
634  if (dest_file != _tar_filelist[subdir].end()) {
635  /* Link to file. Process the link like the destination file. */
636  _tar_filelist[subdir].insert(TarFileList::value_type(src, dest_file->second));
637  } else {
638  /* Destination file not found. Assume 'link to directory'
639  * Append PATHSEPCHAR to 'src' and 'dest' if needed */
640  const std::string src_path = ((*src.rbegin() == PATHSEPCHAR) ? src : src + PATHSEPCHAR);
641  const std::string dst_path = (dest.length() == 0 ? "" : ((*dest.rbegin() == PATHSEPCHAR) ? dest : dest + PATHSEPCHAR));
642  _tar_linklist[subdir].insert(TarLinkList::value_type(src_path, dst_path));
643  }
644 }
645 
646 void FioTarAddLink(const char *src, const char *dest, Subdirectory subdir)
647 {
648  TarAddLink(src, dest, subdir);
649 }
650 
656 static void SimplifyFileName(char *name)
657 {
658  /* Force lowercase */
659  strtolower(name);
660 
661  /* Tar-files always have '/' path-separator, but we want our PATHSEPCHAR */
662 #if (PATHSEPCHAR != '/')
663  for (char *n = name; *n != '\0'; n++) if (*n == '/') *n = PATHSEPCHAR;
664 #endif
665 }
666 
673 {
674  _tar_filelist[sd].clear();
675  _tar_list[sd].clear();
676  uint num = this->Scan(".tar", sd, false);
677  if (sd == BASESET_DIR || sd == NEWGRF_DIR) num += this->Scan(".tar", OLD_DATA_DIR, false);
678  return num;
679 }
680 
681 /* static */ uint TarScanner::DoScan(TarScanner::Mode mode)
682 {
683  DEBUG(misc, 1, "Scanning for tars");
684  TarScanner fs;
685  uint num = 0;
686  if (mode & TarScanner::BASESET) {
687  num += fs.DoScan(BASESET_DIR);
688  }
689  if (mode & TarScanner::NEWGRF) {
690  num += fs.DoScan(NEWGRF_DIR);
691  }
692  if (mode & TarScanner::AI) {
693  num += fs.DoScan(AI_DIR);
694  num += fs.DoScan(AI_LIBRARY_DIR);
695  }
696  if (mode & TarScanner::GAME) {
697  num += fs.DoScan(GAME_DIR);
698  num += fs.DoScan(GAME_LIBRARY_DIR);
699  }
700  if (mode & TarScanner::SCENARIO) {
701  num += fs.DoScan(SCENARIO_DIR);
702  num += fs.DoScan(HEIGHTMAP_DIR);
703  }
704  DEBUG(misc, 1, "Scan complete, found %d files", num);
705  return num;
706 }
707 
714 bool TarScanner::AddFile(Subdirectory sd, const char *filename)
715 {
716  this->subdir = sd;
717  return this->AddFile(filename, 0);
718 }
719 
720 bool TarScanner::AddFile(const char *filename, size_t basepath_length, const char *tar_filename)
721 {
722  /* No tar within tar. */
723  assert(tar_filename == NULL);
724 
725  /* The TAR-header, repeated for every file */
726  struct TarHeader {
727  char name[100];
728  char mode[8];
729  char uid[8];
730  char gid[8];
731  char size[12];
732  char mtime[12];
733  char chksum[8];
734  char typeflag;
735  char linkname[100];
736  char magic[6];
737  char version[2];
738  char uname[32];
739  char gname[32];
740  char devmajor[8];
741  char devminor[8];
742  char prefix[155];
743 
744  char unused[12];
745  };
746 
747  /* Check if we already seen this file */
748  TarList::iterator it = _tar_list[this->subdir].find(filename);
749  if (it != _tar_list[this->subdir].end()) return false;
750 
751  FILE *f = fopen(filename, "rb");
752  /* Although the file has been found there can be
753  * a number of reasons we cannot open the file.
754  * Most common case is when we simply have not
755  * been given read access. */
756  if (f == NULL) return false;
757 
758  const char *dupped_filename = stredup(filename);
759  _tar_list[this->subdir][filename].filename = dupped_filename;
760  _tar_list[this->subdir][filename].dirname = NULL;
761 
762  TarLinkList links;
763 
764  TarHeader th;
765  char buf[sizeof(th.name) + 1], *end;
766  char name[sizeof(th.prefix) + 1 + sizeof(th.name) + 1];
767  char link[sizeof(th.linkname) + 1];
768  char dest[sizeof(th.prefix) + 1 + sizeof(th.name) + 1 + 1 + sizeof(th.linkname) + 1];
769  size_t num = 0, pos = 0;
770 
771  /* Make a char of 512 empty bytes */
772  char empty[512];
773  memset(&empty[0], 0, sizeof(empty));
774 
775  for (;;) { // Note: feof() always returns 'false' after 'fseek()'. Cool, isn't it?
776  size_t num_bytes_read = fread(&th, 1, 512, f);
777  if (num_bytes_read != 512) break;
778  pos += num_bytes_read;
779 
780  /* Check if we have the new tar-format (ustar) or the old one (a lot of zeros after 'link' field) */
781  if (strncmp(th.magic, "ustar", 5) != 0 && memcmp(&th.magic, &empty[0], 512 - offsetof(TarHeader, magic)) != 0) {
782  /* If we have only zeros in the block, it can be an end-of-file indicator */
783  if (memcmp(&th, &empty[0], 512) == 0) continue;
784 
785  DEBUG(misc, 0, "The file '%s' isn't a valid tar-file", filename);
786  fclose(f);
787  return false;
788  }
789 
790  name[0] = '\0';
791 
792  /* The prefix contains the directory-name */
793  if (th.prefix[0] != '\0') {
794  strecpy(name, th.prefix, lastof(name));
795  strecat(name, PATHSEP, lastof(name));
796  }
797 
798  /* Copy the name of the file in a safe way at the end of 'name' */
799  strecat(name, th.name, lastof(name));
800 
801  /* Calculate the size of the file.. for some strange reason this is stored as a string */
802  strecpy(buf, th.size, lastof(buf));
803  size_t skip = strtoul(buf, &end, 8);
804 
805  switch (th.typeflag) {
806  case '\0':
807  case '0': { // regular file
808  /* Ignore empty files */
809  if (skip == 0) break;
810 
811  if (strlen(name) == 0) break;
812 
813  /* Store this entry in the list */
814  TarFileListEntry entry;
815  entry.tar_filename = dupped_filename;
816  entry.size = skip;
817  entry.position = pos;
818 
819  /* Convert to lowercase and our PATHSEPCHAR */
820  SimplifyFileName(name);
821 
822  DEBUG(misc, 6, "Found file in tar: %s (" PRINTF_SIZE " bytes, " PRINTF_SIZE " offset)", name, skip, pos);
823  if (_tar_filelist[this->subdir].insert(TarFileList::value_type(name, entry)).second) num++;
824 
825  break;
826  }
827 
828  case '1': // hard links
829  case '2': { // symbolic links
830  /* Copy the destination of the link in a safe way at the end of 'linkname' */
831  strecpy(link, th.linkname, lastof(link));
832 
833  if (strlen(name) == 0 || strlen(link) == 0) break;
834 
835  /* Convert to lowercase and our PATHSEPCHAR */
836  SimplifyFileName(name);
837  SimplifyFileName(link);
838 
839  /* Only allow relative links */
840  if (link[0] == PATHSEPCHAR) {
841  DEBUG(misc, 1, "Ignoring absolute link in tar: %s -> %s", name, link);
842  break;
843  }
844 
845  /* Process relative path.
846  * Note: The destination of links must not contain any directory-links. */
847  strecpy(dest, name, lastof(dest));
848  char *destpos = strrchr(dest, PATHSEPCHAR);
849  if (destpos == NULL) destpos = dest;
850  *destpos = '\0';
851 
852  char *pos = link;
853  while (*pos != '\0') {
854  char *next = strchr(pos, PATHSEPCHAR);
855  if (next == NULL) {
856  next = pos + strlen(pos);
857  } else {
858  /* Terminate the substring up to the path separator character. */
859  *next++= '\0';
860  }
861 
862  if (strcmp(pos, ".") == 0) {
863  /* Skip '.' (current dir) */
864  } else if (strcmp(pos, "..") == 0) {
865  /* level up */
866  if (dest[0] == '\0') {
867  DEBUG(misc, 1, "Ignoring link pointing outside of data directory: %s -> %s", name, link);
868  break;
869  }
870 
871  /* Truncate 'dest' after last PATHSEPCHAR.
872  * This assumes that the truncated part is a real directory and not a link. */
873  destpos = strrchr(dest, PATHSEPCHAR);
874  if (destpos == NULL) destpos = dest;
875  *destpos = '\0';
876  } else {
877  /* Append at end of 'dest' */
878  if (destpos != dest) destpos = strecpy(destpos, PATHSEP, lastof(dest));
879  destpos = strecpy(destpos, pos, lastof(dest));
880  }
881 
882  if (destpos >= lastof(dest)) {
883  DEBUG(misc, 0, "The length of a link in tar-file '%s' is too large (malformed?)", filename);
884  fclose(f);
885  return false;
886  }
887 
888  pos = next;
889  }
890 
891  /* Store links in temporary list */
892  DEBUG(misc, 6, "Found link in tar: %s -> %s", name, dest);
893  links.insert(TarLinkList::value_type(name, dest));
894 
895  break;
896  }
897 
898  case '5': // directory
899  /* Convert to lowercase and our PATHSEPCHAR */
900  SimplifyFileName(name);
901 
902  /* Store the first directory name we detect */
903  DEBUG(misc, 6, "Found dir in tar: %s", name);
904  if (_tar_list[this->subdir][filename].dirname == NULL) _tar_list[this->subdir][filename].dirname = stredup(name);
905  break;
906 
907  default:
908  /* Ignore other types */
909  break;
910  }
911 
912  /* Skip to the next block.. */
913  skip = Align(skip, 512);
914  if (fseek(f, skip, SEEK_CUR) < 0) {
915  DEBUG(misc, 0, "The file '%s' can't be read as a valid tar-file", filename);
916  fclose(f);
917  return false;
918  }
919  pos += skip;
920  }
921 
922  DEBUG(misc, 1, "Found tar '%s' with " PRINTF_SIZE " new files", filename, num);
923  fclose(f);
924 
925  /* Resolve file links and store directory links.
926  * We restrict usage of links to two cases:
927  * 1) Links to directories:
928  * Both the source path and the destination path must NOT contain any further links.
929  * When resolving files at most one directory link is resolved.
930  * 2) Links to files:
931  * The destination path must NOT contain any links.
932  * The source path may contain one directory link.
933  */
934  for (TarLinkList::iterator link = links.begin(); link != links.end(); link++) {
935  const std::string &src = link->first;
936  const std::string &dest = link->second;
937  TarAddLink(src, dest, this->subdir);
938  }
939 
940  return true;
941 }
942 
950 bool ExtractTar(const char *tar_filename, Subdirectory subdir)
951 {
952  TarList::iterator it = _tar_list[subdir].find(tar_filename);
953  /* We don't know the file. */
954  if (it == _tar_list[subdir].end()) return false;
955 
956  const char *dirname = (*it).second.dirname;
957 
958  /* The file doesn't have a sub directory! */
959  if (dirname == NULL) return false;
960 
961  char filename[MAX_PATH];
962  strecpy(filename, tar_filename, lastof(filename));
963  char *p = strrchr(filename, PATHSEPCHAR);
964  /* The file's path does not have a separator? */
965  if (p == NULL) return false;
966 
967  p++;
968  strecpy(p, dirname, lastof(filename));
969  DEBUG(misc, 8, "Extracting %s to directory %s", tar_filename, filename);
970  FioCreateDirectory(filename);
971 
972  for (TarFileList::iterator it2 = _tar_filelist[subdir].begin(); it2 != _tar_filelist[subdir].end(); it2++) {
973  if (strcmp((*it2).second.tar_filename, tar_filename) != 0) continue;
974 
975  strecpy(p, (*it2).first.c_str(), lastof(filename));
976 
977  DEBUG(misc, 9, " extracting %s", filename);
978 
979  /* First open the file in the .tar. */
980  size_t to_copy = 0;
981  FILE *in = FioFOpenFileTar(&(*it2).second, &to_copy);
982  if (in == NULL) {
983  DEBUG(misc, 6, "Extracting %s failed; could not open %s", filename, tar_filename);
984  return false;
985  }
986 
987  /* Now open the 'output' file. */
988  FILE *out = fopen(filename, "wb");
989  if (out == NULL) {
990  DEBUG(misc, 6, "Extracting %s failed; could not open %s", filename, filename);
991  fclose(in);
992  return false;
993  }
994 
995  /* Now read from the tar and write it into the file. */
996  char buffer[4096];
997  size_t read;
998  for (; to_copy != 0; to_copy -= read) {
999  read = fread(buffer, 1, min(to_copy, lengthof(buffer)), in);
1000  if (read <= 0 || fwrite(buffer, 1, read, out) != read) break;
1001  }
1002 
1003  /* Close everything up. */
1004  fclose(in);
1005  fclose(out);
1006 
1007  if (to_copy != 0) {
1008  DEBUG(misc, 6, "Extracting %s failed; still %i bytes to copy", filename, (int)to_copy);
1009  return false;
1010  }
1011  }
1012 
1013  DEBUG(misc, 9, " extraction successful");
1014  return true;
1015 }
1016 
1017 #if defined(WIN32) || defined(WINCE)
1018 
1023 extern void DetermineBasePaths(const char *exe);
1024 #else /* defined(WIN32) || defined(WINCE) */
1025 
1033 static bool ChangeWorkingDirectoryToExecutable(const char *exe)
1034 {
1035  bool success = false;
1036 #ifdef WITH_COCOA
1037  char *app_bundle = strchr(exe, '.');
1038  while (app_bundle != NULL && strncasecmp(app_bundle, ".app", 4) != 0) app_bundle = strchr(&app_bundle[1], '.');
1039 
1040  if (app_bundle != NULL) app_bundle[0] = '\0';
1041 #endif /* WITH_COCOA */
1042  char *s = const_cast<char *>(strrchr(exe, PATHSEPCHAR));
1043  if (s != NULL) {
1044  *s = '\0';
1045 #if defined(__DJGPP__)
1046  /* If we want to go to the root, we can't use cd C:, but we must use '/' */
1047  if (s[-1] == ':') chdir("/");
1048 #endif
1049  if (chdir(exe) != 0) {
1050  DEBUG(misc, 0, "Directory with the binary does not exist?");
1051  } else {
1052  success = true;
1053  }
1054  *s = PATHSEPCHAR;
1055  }
1056 #ifdef WITH_COCOA
1057  if (app_bundle != NULL) app_bundle[0] = '.';
1058 #endif /* WITH_COCOA */
1059  return success;
1060 }
1061 
1073 {
1074  /* No working directory, so nothing to do. */
1075  if (_searchpaths[SP_WORKING_DIR] == NULL) return false;
1076 
1077  /* Working directory is root, so do nothing. */
1078  if (strcmp(_searchpaths[SP_WORKING_DIR], PATHSEP) == 0) return false;
1079 
1080  /* No personal/home directory, so the working directory won't be that. */
1081  if (_searchpaths[SP_PERSONAL_DIR] == NULL) return true;
1082 
1083  char tmp[MAX_PATH];
1084  seprintf(tmp, lastof(tmp), "%s%s", _searchpaths[SP_WORKING_DIR], PERSONAL_DIR);
1085  AppendPathSeparator(tmp, lastof(tmp));
1086  return strcmp(tmp, _searchpaths[SP_PERSONAL_DIR]) != 0;
1087 }
1088 
1093 void DetermineBasePaths(const char *exe)
1094 {
1095  char tmp[MAX_PATH];
1096 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1097  const char *xdg_data_home = xdgDataHome(NULL);
1098  seprintf(tmp, lastof(tmp), "%s" PATHSEP "%s", xdg_data_home,
1099  PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR);
1100  free(xdg_data_home);
1101 
1102  AppendPathSeparator(tmp, lastof(tmp));
1103  _searchpaths[SP_PERSONAL_DIR_XDG] = stredup(tmp);
1104 #endif
1105 #if defined(__MORPHOS__) || defined(__AMIGA__) || defined(DOS) || defined(OS2) || !defined(WITH_PERSONAL_DIR)
1106  _searchpaths[SP_PERSONAL_DIR] = NULL;
1107 #else
1108 #ifdef __HAIKU__
1109  BPath path;
1110  find_directory(B_USER_SETTINGS_DIRECTORY, &path);
1111  const char *homedir = stredup(path.Path());
1112 #else
1113  /* getenv is highly unsafe; duplicate it as soon as possible,
1114  * or at least before something else touches the environment
1115  * variables in any way. It can also contain all kinds of
1116  * unvalidated data we rather not want internally. */
1117  const char *homedir = getenv("HOME");
1118  if (homedir != NULL) {
1119  homedir = stredup(homedir);
1120  }
1121 
1122  if (homedir == NULL) {
1123  const struct passwd *pw = getpwuid(getuid());
1124  homedir = (pw == NULL) ? NULL : stredup(pw->pw_dir);
1125  }
1126 #endif
1127 
1128  if (homedir != NULL) {
1129  ValidateString(homedir);
1130  seprintf(tmp, lastof(tmp), "%s" PATHSEP "%s", homedir, PERSONAL_DIR);
1131  AppendPathSeparator(tmp, lastof(tmp));
1132 
1133  _searchpaths[SP_PERSONAL_DIR] = stredup(tmp);
1134  free(homedir);
1135  } else {
1136  _searchpaths[SP_PERSONAL_DIR] = NULL;
1137  }
1138 #endif
1139 
1140 #if defined(WITH_SHARED_DIR)
1141  seprintf(tmp, lastof(tmp), "%s", SHARED_DIR);
1142  AppendPathSeparator(tmp, lastof(tmp));
1143  _searchpaths[SP_SHARED_DIR] = stredup(tmp);
1144 #else
1145  _searchpaths[SP_SHARED_DIR] = NULL;
1146 #endif
1147 
1148 #if defined(__MORPHOS__) || defined(__AMIGA__)
1149  _searchpaths[SP_WORKING_DIR] = NULL;
1150 #else
1151  if (getcwd(tmp, MAX_PATH) == NULL) *tmp = '\0';
1152  AppendPathSeparator(tmp, lastof(tmp));
1153  _searchpaths[SP_WORKING_DIR] = stredup(tmp);
1154 #endif
1155 
1157 
1158  /* Change the working directory to that one of the executable */
1160  if (getcwd(tmp, MAX_PATH) == NULL) *tmp = '\0';
1161  AppendPathSeparator(tmp, lastof(tmp));
1162  _searchpaths[SP_BINARY_DIR] = stredup(tmp);
1163  } else {
1164  _searchpaths[SP_BINARY_DIR] = NULL;
1165  }
1166 
1167  if (_searchpaths[SP_WORKING_DIR] != NULL) {
1168  /* Go back to the current working directory. */
1169  if (chdir(_searchpaths[SP_WORKING_DIR]) != 0) {
1170  DEBUG(misc, 0, "Failed to return to working directory!");
1171  }
1172  }
1173 
1174 #if defined(__MORPHOS__) || defined(__AMIGA__) || defined(DOS) || defined(OS2)
1175  _searchpaths[SP_INSTALLATION_DIR] = NULL;
1176 #else
1177  seprintf(tmp, lastof(tmp), "%s", GLOBAL_DATA_DIR);
1178  AppendPathSeparator(tmp, lastof(tmp));
1179  _searchpaths[SP_INSTALLATION_DIR] = stredup(tmp);
1180 #endif
1181 #ifdef WITH_COCOA
1182 extern void cocoaSetApplicationBundleDir();
1183  cocoaSetApplicationBundleDir();
1184 #else
1185  _searchpaths[SP_APPLICATION_BUNDLE_DIR] = NULL;
1186 #endif
1187 }
1188 #endif /* defined(WIN32) || defined(WINCE) */
1189 
1190 const char *_personal_dir;
1191 
1198 void DeterminePaths(const char *exe)
1199 {
1200  DetermineBasePaths(exe);
1201 
1202 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1203  char config_home[MAX_PATH];
1204 
1205  const char *xdg_config_home = xdgConfigHome(NULL);
1206  seprintf(config_home, lastof(config_home), "%s" PATHSEP "%s", xdg_config_home,
1207  PERSONAL_DIR[0] == '.' ? &PERSONAL_DIR[1] : PERSONAL_DIR);
1208  free(xdg_config_home);
1209 
1210  AppendPathSeparator(config_home, lastof(config_home));
1211 #endif
1212 
1213  Searchpath sp;
1214  FOR_ALL_SEARCHPATHS(sp) {
1215  if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
1216  DEBUG(misc, 4, "%s added as search path", _searchpaths[sp]);
1217  }
1218 
1219  char *config_dir;
1220  if (_config_file != NULL) {
1221  config_dir = stredup(_config_file);
1222  char *end = strrchr(config_dir, PATHSEPCHAR);
1223  if (end == NULL) {
1224  config_dir[0] = '\0';
1225  } else {
1226  end[1] = '\0';
1227  }
1228  } else {
1229  char personal_dir[MAX_PATH];
1230  if (FioFindFullPath(personal_dir, lastof(personal_dir), BASE_DIR, "openttd.cfg") != NULL) {
1231  char *end = strrchr(personal_dir, PATHSEPCHAR);
1232  if (end != NULL) end[1] = '\0';
1233  config_dir = stredup(personal_dir);
1234  _config_file = str_fmt("%sopenttd.cfg", config_dir);
1235  } else {
1236 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1237  /* No previous configuration file found. Use the configuration folder from XDG. */
1238  config_dir = config_home;
1239 #else
1240  static const Searchpath new_openttd_cfg_order[] = {
1242  };
1243 
1244  config_dir = NULL;
1245  for (uint i = 0; i < lengthof(new_openttd_cfg_order); i++) {
1246  if (IsValidSearchPath(new_openttd_cfg_order[i])) {
1247  config_dir = stredup(_searchpaths[new_openttd_cfg_order[i]]);
1248  break;
1249  }
1250  }
1251  assert(config_dir != NULL);
1252 #endif
1253  _config_file = str_fmt("%sopenttd.cfg", config_dir);
1254  }
1255  }
1256 
1257  DEBUG(misc, 3, "%s found as config directory", config_dir);
1258 
1259  _highscore_file = str_fmt("%shs.dat", config_dir);
1260  extern char *_hotkeys_file;
1261  _hotkeys_file = str_fmt("%shotkeys.cfg", config_dir);
1262  extern char *_windows_file;
1263  _windows_file = str_fmt("%swindows.cfg", config_dir);
1264 
1265 #if defined(WITH_XDG_BASEDIR) && defined(WITH_PERSONAL_DIR)
1266  if (config_dir == config_home) {
1267  /* We are using the XDG configuration home for the config file,
1268  * then store the rest in the XDG data home folder. */
1269  _personal_dir = _searchpaths[SP_PERSONAL_DIR_XDG];
1270  FioCreateDirectory(_personal_dir);
1271  } else
1272 #endif
1273  {
1274  _personal_dir = config_dir;
1275  }
1276 
1277  /* Make the necessary folders */
1278 #if !defined(__MORPHOS__) && !defined(__AMIGA__) && defined(WITH_PERSONAL_DIR)
1279  FioCreateDirectory(config_dir);
1280  if (config_dir != _personal_dir) FioCreateDirectory(_personal_dir);
1281 #endif
1282 
1283  DEBUG(misc, 3, "%s found as personal directory", _personal_dir);
1284 
1285  static const Subdirectory default_subdirs[] = {
1287  };
1288 
1289  for (uint i = 0; i < lengthof(default_subdirs); i++) {
1290  char *dir = str_fmt("%s%s", _personal_dir, _subdirs[default_subdirs[i]]);
1291  FioCreateDirectory(dir);
1292  free(dir);
1293  }
1294 
1295  /* If we have network we make a directory for the autodownloading of content */
1296  _searchpaths[SP_AUTODOWNLOAD_DIR] = str_fmt("%s%s", _personal_dir, "content_download" PATHSEP);
1297 #ifdef ENABLE_NETWORK
1298  FioCreateDirectory(_searchpaths[SP_AUTODOWNLOAD_DIR]);
1299 
1300  /* Create the directory for each of the types of content */
1302  for (uint i = 0; i < lengthof(dirs); i++) {
1303  char *tmp = str_fmt("%s%s", _searchpaths[SP_AUTODOWNLOAD_DIR], _subdirs[dirs[i]]);
1304  FioCreateDirectory(tmp);
1305  free(tmp);
1306  }
1307 
1308  extern char *_log_file;
1309  _log_file = str_fmt("%sopenttd.log", _personal_dir);
1310 #else /* ENABLE_NETWORK */
1311  /* If we don't have networking, we don't need to make the directory. But
1312  * if it exists we keep it, otherwise remove it from the search paths. */
1313  if (!FileExists(_searchpaths[SP_AUTODOWNLOAD_DIR])) {
1314  free(_searchpaths[SP_AUTODOWNLOAD_DIR]);
1315  _searchpaths[SP_AUTODOWNLOAD_DIR] = NULL;
1316  }
1317 #endif /* ENABLE_NETWORK */
1318 }
1319 
1324 void SanitizeFilename(char *filename)
1325 {
1326  for (; *filename != '\0'; filename++) {
1327  switch (*filename) {
1328  /* The following characters are not allowed in filenames
1329  * on at least one of the supported operating systems: */
1330  case ':': case '\\': case '*': case '?': case '/':
1331  case '<': case '>': case '|': case '"':
1332  *filename = '_';
1333  break;
1334  }
1335  }
1336 }
1337 
1346 void *ReadFileToMem(const char *filename, size_t *lenp, size_t maxsize)
1347 {
1348  FILE *in = fopen(filename, "rb");
1349  if (in == NULL) return NULL;
1350 
1351  fseek(in, 0, SEEK_END);
1352  size_t len = ftell(in);
1353  fseek(in, 0, SEEK_SET);
1354  if (len > maxsize) {
1355  fclose(in);
1356  return NULL;
1357  }
1358  byte *mem = MallocT<byte>(len + 1);
1359  mem[len] = 0;
1360  if (fread(mem, len, 1, in) != 1) {
1361  fclose(in);
1362  free(mem);
1363  return NULL;
1364  }
1365  fclose(in);
1366 
1367  *lenp = len;
1368  return mem;
1369 }
1370 
1377 static bool MatchesExtension(const char *extension, const char *filename)
1378 {
1379  if (extension == NULL) return true;
1380 
1381  const char *ext = strrchr(filename, extension[0]);
1382  return ext != NULL && strcasecmp(ext, extension) == 0;
1383 }
1384 
1394 static uint ScanPath(FileScanner *fs, const char *extension, const char *path, size_t basepath_length, bool recursive)
1395 {
1396  extern bool FiosIsValidFile(const char *path, const struct dirent *ent, struct stat *sb);
1397 
1398  uint num = 0;
1399  struct stat sb;
1400  struct dirent *dirent;
1401  DIR *dir;
1402 
1403  if (path == NULL || (dir = ttd_opendir(path)) == NULL) return 0;
1404 
1405  while ((dirent = readdir(dir)) != NULL) {
1406  const char *d_name = FS2OTTD(dirent->d_name);
1407  char filename[MAX_PATH];
1408 
1409  if (!FiosIsValidFile(path, dirent, &sb)) continue;
1410 
1411  seprintf(filename, lastof(filename), "%s%s", path, d_name);
1412 
1413  if (S_ISDIR(sb.st_mode)) {
1414  /* Directory */
1415  if (!recursive) continue;
1416  if (strcmp(d_name, ".") == 0 || strcmp(d_name, "..") == 0) continue;
1417  if (!AppendPathSeparator(filename, lastof(filename))) continue;
1418  num += ScanPath(fs, extension, filename, basepath_length, recursive);
1419  } else if (S_ISREG(sb.st_mode)) {
1420  /* File */
1421  if (MatchesExtension(extension, filename) && fs->AddFile(filename, basepath_length, NULL)) num++;
1422  }
1423  }
1424 
1425  closedir(dir);
1426 
1427  return num;
1428 }
1429 
1436 static uint ScanTar(FileScanner *fs, const char *extension, TarFileList::iterator tar)
1437 {
1438  uint num = 0;
1439  const char *filename = (*tar).first.c_str();
1440 
1441  if (MatchesExtension(extension, filename) && fs->AddFile(filename, 0, (*tar).second.tar_filename)) num++;
1442 
1443  return num;
1444 }
1445 
1455 uint FileScanner::Scan(const char *extension, Subdirectory sd, bool tars, bool recursive)
1456 {
1457  this->subdir = sd;
1458 
1459  Searchpath sp;
1460  char path[MAX_PATH];
1461  TarFileList::iterator tar;
1462  uint num = 0;
1463 
1464  FOR_ALL_SEARCHPATHS(sp) {
1465  /* Don't search in the working directory */
1466  if (sp == SP_WORKING_DIR && !_do_scan_working_directory) continue;
1467 
1468  FioAppendDirectory(path, lastof(path), sp, sd);
1469  num += ScanPath(this, extension, path, strlen(path), recursive);
1470  }
1471 
1472  if (tars && sd != NO_DIRECTORY) {
1473  FOR_ALL_TARS(tar, sd) {
1474  num += ScanTar(this, extension, tar);
1475  }
1476  }
1477 
1478  switch (sd) {
1479  case BASESET_DIR:
1480  num += this->Scan(extension, OLD_GM_DIR, tars, recursive);
1481  /* FALL THROUGH */
1482  case NEWGRF_DIR:
1483  num += this->Scan(extension, OLD_DATA_DIR, tars, recursive);
1484  break;
1485 
1486  default: break;
1487  }
1488 
1489  return num;
1490 }
1491 
1500 uint FileScanner::Scan(const char *extension, const char *directory, bool recursive)
1501 {
1502  char path[MAX_PATH];
1503  strecpy(path, directory, lastof(path));
1504  if (!AppendPathSeparator(path, lastof(path))) return 0;
1505  return ScanPath(this, extension, path, strlen(path), recursive);
1506 }